Workday Connector - adding some info re skipped records (#1616)

faros-ai · Jul 31, 2024 · 5f509b2 · 5f509b2
1 parent dc12221
commit 5f509b2
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 0 deletions.
diff --git a/destinations/airbyte-faros-destination/src/converters/workday/customreports.ts b/destinations/airbyte-faros-destination/src/converters/workday/customreports.ts
@@ -45,6 45,9 @@ export class Customreports extends Converter {
  // These are set in setOrgsToKeepAndIgnore
  org_ids_to_keep = null;
  org_ids_to_ignore = null;
  skipped_due_to_missing_fields = 0;
  skipped_due_to_termination = 0;
  failedRecordFields: Set<string> = new Set<string>();
 
  /** Almost every SquadCast record have id property */
  id(record: AirbyteRecord): any {
@@ -170,19 173,41 @@ export class Customreports extends Converter {
  !rec.Team_ID ||
  !rec.Team_Name
  ) {
  // We convert the record keys into a sorted comma-separated string
  this.failedRecordFields.add(this.getSortedRecordFields(rec));
  this.skipped_due_to_missing_fields  = 1;
  return false;
  }
  // We're only keeping active records
  if (this.isTerminated(rec)) {
  if (
  !ctx.config.source_specific_configs?.workday?.keep_terminated_employees
  ) {
  this.skipped_due_to_termination  = 1;
  return false;
  }
  }
  return true;
  }
 
  private getSortedRecordFields(rec: EmployeeRecord): string {
  const record_keys: string[] = [];
  for (const key in rec) {
  // Check if type is string:
  if (typeof key === 'string') {
  record_keys.push(key);
  } else {
  throw new Error(
  `Key ${key} is not a string, instead type: ${typeof key}`
  );
  }
  }
  const sorted_keys_str = record_keys
  .sort((a, b) => a.localeCompare(b, undefined, {sensitivity: 'base'}))
  .join(',');
  return sorted_keys_str;
  }
 
  private getManagerIDFromList(recs: ManagerTimeRecord[]): string {
  if (!recs) {
  throw new Error('Missing recs');
@@ -366,6 391,9 @@ export class Customreports extends Converter {
  nAcceptableTeams: acceptableTeams.size,
  nOriginalTeams: teamIDs ? teamIDs.length : 0,
  records_skipped: this.recordCount.skippedRecords,
  numRecordsSkippedDueToMissingFields: this.skipped_due_to_missing_fields,
  failedRecordFields: Array.from(this.failedRecordFields.values()),
  numRecordsSkippedDueToTermination: this.skipped_due_to_termination,
  records_stored: this.recordCount.storedRecords,
  nCycleChains: this.cycleChains ? this.cycleChains.length : 0,
  generalLogs: this.generalLogCollection,

diff --git a/destinations/airbyte-faros-destination/test/converters/__snapshots__/workday.test.ts.snap b/destinations/airbyte-faros-destination/test/converters/__snapshots__/workday.test.ts.snap
@@ -1,5 1,16 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`workday Ignore improperly formatted records from customreports v6 stream 1`] = `
 Array [
  "Processed 4 records",
  "Processed records by stream: {\\\\\\"mytestsource__workday__customreports\\\\\\":4}\\"},\\"type\\":\\"LOG\\"}",
  "Would write 13 records",
  "Would write records by model: {\\\\\\"geo_Location\\\\\\":1,\\\\\\"identity_Identity\\\\\\":3,\\\\\\"org_Employee\\\\\\":3,\\\\\\"org_Team\\\\\\":3,\\\\\\"org_TeamMembership\\\\\\":3}\\"},\\"type\\":\\"LOG\\"}",
  "Skipped 0 records",
  "Errored 0 records",
 ]
 `;
 
 exports[`workday Randomly generated records from customreports v4 stream with Terminated 1`] = `
 Array [
  "Processed 103 records",

diff --git a/destinations/airbyte-faros-destination/test/converters/workday.test.ts b/destinations/airbyte-faros-destination/test/converters/workday.test.ts
@@ -173,6 173,14 @@ describe('workday', () => {
  inputRecordsPath: 'workday/stream_v4.log',
  });
  });
  test('Ignore improperly formatted records from customreports v6 stream', async () => {
  const configPath = await getTempConfig([], []);
  await destinationWriteTest({
  configPath,
  catalogPath: 'test/resources/workday/catalog.json',
  inputRecordsPath: 'workday/stream_v6.log',
  });
  });
  test('check resulting org structure from "empty" input', () => {
  const [customReportDestination, ctx] = getCustomReportandCtxGivenKey(
  mockttp,

diff --git a/destinations/airbyte-faros-destination/test/resources/workday/stream_v6.log b/destinations/airbyte-faros-destination/test/resources/workday/stream_v6.log
@@ -0,0 1,4 @@
 {"record": {"stream": "mytestsource__workday__customreports", "emitted_at": 1697757823488, "data": {"Start_Date": "2021-04-28T10:41:28.768Z", "Full_Name": "James Rodriguez", "Employee_ID": "100", "Termination_Date": null, "Location": "New York", "Email": "[email protected]", "Team_Name": "Team C", "Manager_ID": "105", "Manager_Name": "Olivia Johnson", "Team_ID": "C", "Job_Title": "Crayon Supplier"}}, "type": "RECORD"}
 {"record": {"stream": "mytestsource__workday__customreports", "emitted_at": 1697757823488, "data": {"Start_Date": "2018-08-07T11:41:53.577Z", "Full_Name": "Charlotte Miller", "Employee_ID": "101", "Termination_Date": null, "Location": "New York", "Email": "[email protected]", "Team_Name": "Team B", "Manager_ID": "149", "Manager_Name": "Lucas Brown", "Team_ID": "B", "Employee_Type": "FullTime"}}, "type": "RECORD"}
 {"record": {"stream": "mytestsource__workday__customreports", "emitted_at": 1697757823488, "data": {"startDate": "2020-04-10T08:18:58.882Z", "FullName": "Isabella Smith", "Employee_ID": "113", "Termination_Date": null, "Location": "New York", "Email": "[email protected]", "Team_Name": "Team TopDog", "Manager_ID": "140", "Manager_Name": "Elijah Davis", "Team_ID": "TopDog"}}, "type": "RECORD"}
 {"record": {"stream": "mytestsource__workday__customreports", "emitted_at": 1697745978747, "data": {"Start_Date": "2017-08-31T22:06:43.488Z", "Manager_Name": "John White", "Manager_ID": "297", "Team_Name": "Also doesn't matter", "Termination_date": "2050-08-31T22:06:43.488Z", "Location": "Texas", "Email": "[email protected]", "Team_ID": "H"}}, "type": "RECORD"}