From ea4c4354a6ec167f712d7ee297f9dc606edd9eff Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Fri, 10 Apr 2026 09:43:39 -0400 Subject: [PATCH 1/2] Fixes for communicates with options --- src/commands/org_data/index.ts | 11 +- .../integrations/azure_integration.ts | 66 ++++--- .../integrations/cloudtrail_integration.ts | 186 +++++++++++++++++- .../integrations/jamf_pro_integration.ts | 9 +- src/commands/org_data/org_data.ts | 10 +- src/constants.ts | 14 +- src/utils/kibana_api.ts | 51 +++-- 7 files changed, 286 insertions(+), 61 deletions(-) diff --git a/src/commands/org_data/index.ts b/src/commands/org_data/index.ts index f1f971be..160db9e7 100644 --- a/src/commands/org_data/index.ts +++ b/src/commands/org_data/index.ts @@ -20,17 +20,26 @@ export const orgDataCommands: CommandModule = { ) .option('--all', 'Generate all integrations regardless of company size') .option('--detection-rules', 'Include sample detection rules for applicable integrations') + .option( + '--size ', + 'Organization size without prompting (john-doe|small|medium|enterprise)', + ) + .option( + '--productivity-suite ', + 'Productivity suite without prompting (microsoft|google)', + ) .addHelpText('after', '\n' + getOrgDataHelp()) .action( wrapAction(async (options) => { await runOrgData({ - size: 'medium', + size: options.size, name: options.name, space: options.space, seed: options.seed, integrations: options.integrations, all: options.all, detectionRules: options.detectionRules, + productivitySuite: options.productivitySuite, }); }), ); diff --git a/src/commands/org_data/integrations/azure_integration.ts b/src/commands/org_data/integrations/azure_integration.ts index c695575e..3d64b25b 100644 --- a/src/commands/org_data/integrations/azure_integration.ts +++ b/src/commands/org_data/integrations/azure_integration.ts @@ -450,29 +450,48 @@ export class AzureIntegration extends BaseIntegration { const auditCount = Math.max(5, Math.ceil(employees.length * 0.3)); for (let i = 0; i < auditCount; i++) { - const employee = faker.helpers.arrayElement(employees); - docs.push(this.createAuditLogDoc(employee, tenantId)); + const actor = faker.helpers.arrayElement(employees); + docs.push(this.createAuditLogDoc(actor, employees, tenantId)); } return docs; } - private createAuditLogDoc(employee: Employee, tenantId: string): IntegrationDocument { + private createAuditLogDoc( + actor: Employee, + employees: Employee[], + tenantId: string, + ): IntegrationDocument { const timestamp = this.getRandomTimestamp(72); const activity = faker.helpers.weightedArrayElement( AUDIT_LOG_ACTIVITIES.map((a) => ({ value: a, weight: a.weight })), ); const correlationId = faker.string.uuid(); const targetId = faker.string.uuid(); - const targetDisplayName = - activity.targetType === 'User' - ? `${employee.firstName} ${employee.lastName}` - : activity.targetType === 'Device' - ? `DESKTOP-${faker.string.alphanumeric(7).toUpperCase()}` - : `${activity.category}-${faker.string.alphanumeric(6)}`; - const isAppInitiated = faker.datatype.boolean(); + // Pick a distinct target employee for User-type relationships + const otherEmployees = employees.filter((e) => e.email !== actor.email); + const targetEmployee = + otherEmployees.length > 0 ? faker.helpers.arrayElement(otherEmployees) : actor; + + // Build the target resource. User-type targets need userPrincipalName so the + // ingest pipeline maps it to target_resources.0.user_principal_name, which is + // what the communicates_with ES|QL query reads for user→user relationships. + const targetResource: Record = { + id: targetId, + type: activity.targetType, + }; + if (activity.targetType === 'User') { + targetResource.displayName = `${targetEmployee.firstName} ${targetEmployee.lastName}`; + targetResource.userPrincipalName = targetEmployee.email; + } else if (activity.targetType === 'Device') { + targetResource.displayName = `DESKTOP-${faker.string.alphanumeric(7).toUpperCase()}`; + } else { + targetResource.displayName = `${activity.category}-${faker.string.alphanumeric(6)}`; + } + // Always user-initiated so the actor UPN (initiated_by.user.userPrincipalName) + // is present and can be used to build the actor EUID (user:{email}@entra_id). const rawAzureJson = { time: timestamp, category: 'AuditLogs', @@ -491,27 +510,14 @@ export class AzureIntegration extends BaseIntegration { loggedByService: activity.loggedByService, operationType: activity.operationType, resultReason: '', - targetResources: [ - { - displayName: targetDisplayName, - id: targetId, - type: activity.targetType, + targetResources: [targetResource], + initiatedBy: { + user: { + displayName: `${actor.firstName} ${actor.lastName}`, + userPrincipalName: actor.email, + id: actor.entraIdUserId, }, - ], - initiatedBy: isAppInitiated - ? { - app: { - displayName: 'Device Registration Service', - servicePrincipalId: faker.string.uuid(), - }, - } - : { - user: { - displayName: `${employee.firstName} ${employee.lastName}`, - userPrincipalName: employee.email, - id: employee.entraIdUserId, - }, - }, + }, }, }; diff --git a/src/commands/org_data/integrations/cloudtrail_integration.ts b/src/commands/org_data/integrations/cloudtrail_integration.ts index adc6e7af..b78fc219 100644 --- a/src/commands/org_data/integrations/cloudtrail_integration.ts +++ b/src/commands/org_data/integrations/cloudtrail_integration.ts @@ -25,7 +25,16 @@ import { faker } from '@faker-js/faker'; const AWS_API_EVENTS: Record = { EC2: { eventSource: 'ec2.amazonaws.com', - events: ['DescribeInstances', 'DescribeSecurityGroups', 'DescribeVpcs', 'DescribeSubnets'], + events: [ + 'DescribeInstances', + 'DescribeSecurityGroups', + 'DescribeVpcs', + 'DescribeSubnets', + 'RunInstances', + 'StopInstances', + 'StartInstances', + 'TerminateInstances', + ], }, S3: { eventSource: 's3.amazonaws.com', @@ -87,6 +96,68 @@ const AWS_USER_AGENTS = [ 'terraform-aws-provider/5.31.0', ]; +/** + * Tags required on every document for the ingest pipeline's entity + * classification script to run. Without 'actor_target_mapping' the + * Painless script that populates host.target.entity.id, user.entity.id, + * etc. is skipped entirely. + */ +const CLOUDTRAIL_TAGS = ['actor_target_mapping', 'forwarded']; + +/** + * Events the ingest pipeline's enrichment functions map to enrichCtx.target + * using the instance ID, which the target classifier then routes to + * host.target.entity.id (and host.id) via the i- prefix. + * + * Each entry carries its eventSource and the raw-event parameter shape + * the pipeline reads the instance ID from. + */ +const HOST_TARGET_EVENTS: { + eventSource: string; + eventName: string; + buildParams: (instanceId: string) => { + requestParameters: Record; + responseElements?: null; + }; +}[] = [ + { + eventSource: 'ec2.amazonaws.com', + eventName: 'GetPasswordData', + buildParams: (id) => ({ + requestParameters: { instanceId: id }, + responseElements: null, + }), + }, + { + eventSource: 'ec2-instance-connect.amazonaws.com', + eventName: 'SendSSHPublicKey', + buildParams: (id) => ({ + requestParameters: { + instanceId: id, + instanceOSUser: 'ec2-user', + sSHPublicKey: 'REDACTED', + }, + }), + }, + { + eventSource: 'ssm.amazonaws.com', + eventName: 'SendCommand', + buildParams: (id) => ({ + requestParameters: { + instanceIds: [id], + documentName: 'AWS-RunShellScript', + }, + }), + }, + { + eventSource: 'ssm.amazonaws.com', + eventName: 'StartSession', + buildParams: (id) => ({ + requestParameters: { target: id }, + }), + }, +]; + /** * AWS CloudTrail Integration */ @@ -249,6 +320,46 @@ export class CloudTrailIntegration extends BaseIntegration { ), ); } + + // Generate events that produce host.target.entity.id (2-4 per session) + const ec2Instances = accountResources.filter((r) => r.subType === 'AWS EC2 Instance'); + if (ec2Instances.length > 0) { + const instanceEventCount = faker.number.int({ min: 2, max: 4 }); + for (let i = 0; i < instanceEventCount; i++) { + const apiTime = new Date( + new Date(sessionStart).getTime() + faker.number.int({ min: 30, max: 600 }) * 1000, + ).toISOString(); + const targetInstance = faker.helpers.arrayElement(ec2Instances); + + events.push( + this.createHostTargetEvent( + { + type: 'AssumedRole', + principalId: `${this.generateRoleId()}:${employee.email}`, + arn: assumedRoleArn, + accountId: account!.id, + accessKeyId, + sessionContext: { + attributes: { + mfaAuthenticated: 'true', + creationDate: apiTime, + }, + sessionIssuer: { + type: 'Role', + principalId: this.generateRoleId(), + arn: `arn:aws:iam::${account!.id}:role/okta`, + accountId: account!.id, + userName: 'okta', + }, + }, + }, + account!, + apiTime, + targetInstance, + ), + ); + } + } } return events; @@ -286,6 +397,32 @@ export class CloudTrailIntegration extends BaseIntegration { ); } + // Generate events that produce host.target.entity.id (2-4 per batch) + const ec2Instances = accountResources.filter((r) => r.subType === 'AWS EC2 Instance'); + if (ec2Instances.length > 0) { + const instanceEventCount = faker.number.int({ min: 2, max: 4 }); + for (let i = 0; i < instanceEventCount; i++) { + const timestamp = this.getRandomTimestamp(48); + const targetInstance = faker.helpers.arrayElement(ec2Instances); + + events.push( + this.createHostTargetEvent( + { + type: 'IAMUser', + principalId: this.generatePrincipalId(), + arn: iamUser.arn, + accountId: account!.id, + accessKeyId, + userName: iamUser.userName, + }, + account!, + timestamp, + targetInstance, + ), + ); + } + } + return events; } @@ -478,6 +615,7 @@ export class CloudTrailIntegration extends BaseIntegration { return { '@timestamp': timestamp, message: JSON.stringify(rawEvent), + tags: CLOUDTRAIL_TAGS, data_stream: { namespace: 'default', type: 'logs', dataset: 'aws.cloudtrail' }, } as IntegrationDocument; } @@ -537,6 +675,7 @@ export class CloudTrailIntegration extends BaseIntegration { return { '@timestamp': timestamp, message: JSON.stringify(rawEvent), + tags: CLOUDTRAIL_TAGS, data_stream: { namespace: 'default', type: 'logs', dataset: 'aws.cloudtrail' }, } as IntegrationDocument; } @@ -617,6 +756,7 @@ export class CloudTrailIntegration extends BaseIntegration { return { '@timestamp': timestamp, message: JSON.stringify(rawEvent), + tags: CLOUDTRAIL_TAGS, data_stream: { namespace: 'default', type: 'logs', dataset: 'aws.cloudtrail' }, } as IntegrationDocument; } @@ -670,6 +810,7 @@ export class CloudTrailIntegration extends BaseIntegration { return { '@timestamp': timestamp, message: JSON.stringify(rawEvent), + tags: CLOUDTRAIL_TAGS, data_stream: { namespace: 'default', type: 'logs', dataset: 'aws.cloudtrail' }, } as IntegrationDocument; } @@ -737,6 +878,49 @@ export class CloudTrailIntegration extends BaseIntegration { return { '@timestamp': timestamp, message: JSON.stringify(rawEvent), + tags: CLOUDTRAIL_TAGS, + data_stream: { namespace: 'default', type: 'logs', dataset: 'aws.cloudtrail' }, + } as IntegrationDocument; + } + + /** + * Create an event that the ingest pipeline maps to host.target.entity.id. + * Uses HOST_TARGET_EVENTS which produce the exact parameter shapes that + * enrichEc2 / enrichSsm / enrichEc2InstanceConnect add to enrichCtx.target. + */ + private createHostTargetEvent( + userIdentity: Record, + account: CloudAccount, + timestamp: string, + instance: CloudResource, + ): IntegrationDocument { + const hostTargetEvent = faker.helpers.arrayElement(HOST_TARGET_EVENTS); + const params = hostTargetEvent.buildParams(instance.id); + + const rawEvent: Record = { + eventVersion: '1.08', + userIdentity, + eventTime: timestamp, + eventSource: hostTargetEvent.eventSource, + eventName: hostTargetEvent.eventName, + awsRegion: instance.region, + sourceIPAddress: faker.internet.ipv4(), + userAgent: faker.helpers.arrayElement(AWS_USER_AGENTS), + requestParameters: params.requestParameters, + ...(params.responseElements !== undefined && { + responseElements: params.responseElements, + }), + eventID: faker.string.uuid(), + readOnly: hostTargetEvent.eventName === 'GetPasswordData', + eventType: 'AwsApiCall', + managementEvent: true, + recipientAccountId: account.id, + }; + + return { + '@timestamp': timestamp, + message: JSON.stringify(rawEvent), + tags: CLOUDTRAIL_TAGS, data_stream: { namespace: 'default', type: 'logs', dataset: 'aws.cloudtrail' }, } as IntegrationDocument; } diff --git a/src/commands/org_data/integrations/jamf_pro_integration.ts b/src/commands/org_data/integrations/jamf_pro_integration.ts index 6e9d54fe..f57d7840 100644 --- a/src/commands/org_data/integrations/jamf_pro_integration.ts +++ b/src/commands/org_data/integrations/jamf_pro_integration.ts @@ -129,7 +129,7 @@ export class JamfProIntegration extends BaseIntegration { private createInventoryDocument( device: Device, employee: Employee, - org: Organization, + _org: Organization, udid: string, jssId: string, ): IntegrationDocument { @@ -138,14 +138,17 @@ export class JamfProIntegration extends BaseIntegration { const lastReportDate = faker.date.recent({ days: 3 }).toISOString(); const lastEnrolledDate = faker.date.past({ years: 1 }).toISOString(); const initialEntryDate = faker.date.past({ years: 2 }).toISOString().split('T')[0]; - const ipAddress = `10.${faker.number.int({ min: 0, max: 255 })}.${faker.number.int({ min: 0, max: 255 })}.${faker.number.int({ min: 1, max: 254 })}`; const managementId = faker.string.uuid(); + // Use device IP so entity store can correlate with endpoint host.ip + const ipAddress = device.ipAddress; + // Use same hostname pattern as endpoint (employee.userName-platform) for host entity correlation + const hostname = `${employee.userName}-mac`; const inventoryPayload = { id: jssId, udid: udid, general: { - name: `${org.name.toLowerCase().replace(/[^a-z0-9]/g, '')}-${device.serialNumber}`, + name: hostname, platform: 'Mac', last_ip_address: ipAddress, last_reported_ip: ipAddress, diff --git a/src/commands/org_data/org_data.ts b/src/commands/org_data/org_data.ts index 0833f0be..dcd4f065 100644 --- a/src/commands/org_data/org_data.ts +++ b/src/commands/org_data/org_data.ts @@ -177,13 +177,13 @@ const runOrgDataHelper = async ( export const runOrgData = async (options: OrganizationOptions): Promise => { log.info('\n=== Correlated Organization Data Generator ===\n'); - // Prompt for organization size - const size = await promptForSize(); + // Prompt for organization size only if not provided via CLI + const size = options.size ?? (await promptForSize()); - // Prompt for productivity suite - const productivitySuite = await promptForProductivitySuite(); + // Prompt for productivity suite only if not provided via CLI + const productivitySuite = options.productivitySuite ?? (await promptForProductivitySuite()); - // Prompt for detection rules if not set via CLI flag + // Prompt for detection rules only if not provided via CLI const includeDetectionRules = options.detectionRules ?? (await confirm({ diff --git a/src/constants.ts b/src/constants.ts index f6540eff..d5d10816 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -85,13 +85,13 @@ export const ASSET_CRITICALITY_BULK_URL = '/api/asset_criticality/bulk'; export const DETECTION_ENGINE_RULES_URL = '/api/detection_engine/rules'; export const DETECTION_ENGINE_RULES_BULK_ACTION_URL = `${DETECTION_ENGINE_RULES_URL}/_bulk_action`; export const COMPONENT_TEMPLATES_URL = '/api/index_management/component_templates'; -export const FLEET_EPM_PACKAGES_URL = (packageName: string, version: string = 'latest') => { - let url = `/api/fleet/epm/packages/${packageName}`; - if (version !== 'latest') { - url = `${url}/${version}`; - } - return url; -}; +/** GET package metadata — path is /packages/{pkgName} only (Kibana Fleet API). */ +export const FLEET_EPM_PACKAGES_URL = (packageName: string) => + `/api/fleet/epm/packages/${packageName}`; + +/** POST install from registry — {pkgVersion} must be a semver (resolve “latest” via GET package first). */ +export const FLEET_EPM_INSTALL_PACKAGE_URL = (packageName: string, version: string) => + `/api/fleet/epm/packages/${packageName}/${encodeURIComponent(version)}`; export const SPACES_URL = '/api/spaces/space'; export const SPACE_URL = (space: string) => `/api/spaces/space/${space}`; diff --git a/src/utils/kibana_api.ts b/src/utils/kibana_api.ts index c2dc23e5..3ad1cabb 100644 --- a/src/utils/kibana_api.ts +++ b/src/utils/kibana_api.ts @@ -10,6 +10,7 @@ import { DETECTION_ENGINE_RULES_URL, COMPONENT_TEMPLATES_URL, FLEET_EPM_PACKAGES_URL, + FLEET_EPM_INSTALL_PACKAGE_URL, SPACES_URL, SPACE_URL, RISK_SCORE_URL, @@ -151,17 +152,31 @@ export const kibanaFetch = async ( throw new Error(message, { cause: error }); } const rawResponse = await result.text(); - // log response status + let data: unknown; - try { - data = rawResponse ? JSON.parse(rawResponse) : {}; - } catch { - data = { message: rawResponse }; + if (!rawResponse.trim()) { + data = {}; + } else { + try { + data = JSON.parse(rawResponse); + } catch { + const origin = (() => { + try { + return new URL(url).origin; + } catch { + return '(invalid kibana URL)'; + } + })(); + throw new Error( + `Kibana API returned non-JSON (HTTP ${result.status}) for ${path} at ${origin}. ` + + `Body starts with: ${JSON.stringify(rawResponse.slice(0, 120))}. ` + + `Check kibana.node is the Kibana base URL (not Elasticsearch). If Kibana uses server.basePath, ` + + `include it in kibana.node (e.g. http://host:5601/mybase).`, + ); + } } if (!data || typeof data !== 'object') { - throw new Error( - `Unexpected non-object response from ${method} ${safeUrl}. Raw response: ${rawResponse.slice(0, 500)}`, - ); + throw new Error(`Unexpected Kibana response shape for ${path}`); } if (result.status >= 400 && !ignoreStatusesArray.includes(result.status)) { @@ -342,7 +357,13 @@ export const installPackage = async ({ space?: string; prerelease?: boolean; }) => { - let url = FLEET_EPM_PACKAGES_URL(packageName, version); + let resolvedVersion = version; + if (version === 'latest') { + const pkg = await getPackageInfo({ packageName, space, prerelease }); + resolvedVersion = pkg.item.version; + } + + let url = FLEET_EPM_INSTALL_PACKAGE_URL(packageName, resolvedVersion); if (prerelease) { url += '?prerelease=true'; } @@ -392,15 +413,17 @@ export const createAgentPolicy = async ({ export const getPackageInfo = async ({ packageName, space, + prerelease = false, }: { packageName: string; space?: string; + prerelease?: boolean; }): Promise<{ item: { name: string; version: string; status: string } }> => { - return kibanaFetch( - FLEET_EPM_PACKAGES_URL(packageName), - { method: 'GET' }, - { apiVersion: API_VERSIONS.public.v1, space }, - ); + let path = FLEET_EPM_PACKAGES_URL(packageName); + if (prerelease) { + path += '?prerelease=true'; + } + return kibanaFetch(path, { method: 'GET' }, { apiVersion: API_VERSIONS.public.v1, space }); }; export const getPackagePolicies = async ({ From 850f417bdf8da59d25c891e4fa3ef85c903facd9 Mon Sep 17 00:00:00 2001 From: Sean Rathier Date: Fri, 10 Apr 2026 09:47:47 -0400 Subject: [PATCH 2/2] docs: document org-data non-interactive options in README Co-Authored-By: Claude Sonnet 4.6 --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index 6ecf250b..e2a0e82c 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,39 @@ yarn start privmon-quick --space default yarn start csp --data-sources all --findings-count 50 ``` +### Correlated Organization Data + +Interactive — prompts for size, productivity suite, and detection rules: + +```bash +yarn start org-data +``` + +Non-interactive — skip all prompts by providing options directly: + +```bash +yarn start org-data --size small --productivity-suite microsoft --integrations azure,entra_id +``` + +Enable detection rules without prompting: + +```bash +yarn start org-data --size medium --detection-rules +``` + +Each prompt is skipped individually when its flag is present. Omit any flag to be prompted for it: + +| Flag | Values | Default (when omitted) | +| ---------------------- | ------------------------------------------- | ------------------------ | +| `--size` | `john-doe`, `small`, `medium`, `enterprise` | interactive prompt | +| `--productivity-suite` | `microsoft`, `google` | interactive prompt | +| `--detection-rules` | flag (boolean) | interactive prompt | +| `--integrations` | comma-separated list | all default integrations | +| `--all` | flag (boolean) | — | +| `--name` | string | `Acme CRM` | +| `--space` | string | `default` | +| `--seed` | number | random | + ## Commands Detailed command documentation is colocated with command code under `src/commands`.