From 9b436ec9babbec1088e8cb050c3d693af22ec486 Mon Sep 17 00:00:00 2001 From: Yaroslav Serhieiev Date: Tue, 3 Dec 2024 09:26:26 +0200 Subject: [PATCH] feat: third-party import --- .../core/src/structures/synset/parseSynset.ts | 2 +- packages/engine/src/cli/synsets-cmd/argv.ts | 1 + packages/engine/src/cli/synsets-cmd/common.ts | 27 ++++++++++------- packages/engine/src/cli/synsets-cmd/pull.ts | 14 ++++++++- packages/engine/src/cli/synsets-cmd/push.ts | 4 +++ packages/engine/src/cli/synsets.ts | 4 +++ packages/engine/src/sync/words/GSheetsOp.ts | 29 ++++++++++++++++--- packages/engine/src/sync/words/Git2Gsheets.ts | 14 ++++----- 8 files changed, 72 insertions(+), 23 deletions(-) diff --git a/packages/core/src/structures/synset/parseSynset.ts b/packages/core/src/structures/synset/parseSynset.ts index 0d66ce3..82a5093 100644 --- a/packages/core/src/structures/synset/parseSynset.ts +++ b/packages/core/src/structures/synset/parseSynset.ts @@ -3,7 +3,7 @@ import { Lemma } from '../lemma'; import { isVerified, stripMetacharacters } from './metacharacters'; export function parseSynset(rawString: string) { - const sanitized = sanitize(rawString); + const sanitized = sanitize(rawString || ''); const verified = isVerified(sanitized); const annotations = sanitized.includes('(') ? new AnnotationHelper() diff --git a/packages/engine/src/cli/synsets-cmd/argv.ts b/packages/engine/src/cli/synsets-cmd/argv.ts index 4dd2f95..de80cfe 100644 --- a/packages/engine/src/cli/synsets-cmd/argv.ts +++ b/packages/engine/src/cli/synsets-cmd/argv.ts @@ -5,6 +5,7 @@ export type PullArgv = { subcommand: 'pull'; partial: boolean; only: boolean; + source: string; _: string[]; }; diff --git a/packages/engine/src/cli/synsets-cmd/common.ts b/packages/engine/src/cli/synsets-cmd/common.ts index 3970f99..37c8ff2 100644 --- a/packages/engine/src/cli/synsets-cmd/common.ts +++ b/packages/engine/src/cli/synsets-cmd/common.ts @@ -5,33 +5,40 @@ import type { WordsAddLangSheet, WordsSheet } from '../../google'; export * from './argv'; -export async function getGoogleGitSyncPrerequisites() { +interface Overrides { + spreadsheetName?: string; + mainLanguagesSheet?: string; + additionalLanguagesSheet?: string; +} + +export async function getGoogleGitSyncPrerequisites({ + spreadsheetName = 'new_interslavic_words_list', + mainLanguagesSheet = 'words', + additionalLanguagesSheet = 'words_add_lang' +}: Overrides = {}) { const { fileDatabase, googleAPIs } = await compose(); const spreadsheetConfig = await fileDatabase.spreadsheets.findById( - 'new_interslavic_words_list', + spreadsheetName, ); if (!spreadsheetConfig) { throw new Error( - 'Cannot find the spreadsheet config for "new_interslavic_words_list"', + `Cannot find the spreadsheet config with the name: ${spreadsheetName}`, ); } const spreadsheet = googleAPIs.spreadsheet(spreadsheetConfig.google_id); - const words = await spreadsheet.getSheetByTitle('words'); + const words = await spreadsheet.getSheetByTitle(mainLanguagesSheet); if (!words) { - throw new Error('Cannot find the sheet: words'); + throw new Error(`Cannot find the sheet: ${mainLanguagesSheet}`); } - const wordsAddLang = await spreadsheet.getSheetByTitle('words_add_lang'); - if (!wordsAddLang) { - throw new Error('Cannot find the sheet: words_add_lang'); - } + const wordsAddLang = await spreadsheet.getSheetByTitle(additionalLanguagesSheet); return { words: words as unknown as WordsSheet, - wordsAddLang: wordsAddLang as unknown as WordsAddLangSheet, + wordsAddLang: wordsAddLang as unknown as (WordsAddLangSheet | undefined), multisynsets: fileDatabase.multisynsets, }; } diff --git a/packages/engine/src/cli/synsets-cmd/pull.ts b/packages/engine/src/cli/synsets-cmd/pull.ts index f6730e8..0dd4324 100644 --- a/packages/engine/src/cli/synsets-cmd/pull.ts +++ b/packages/engine/src/cli/synsets-cmd/pull.ts @@ -3,9 +3,21 @@ import { GSheets2Git } from '../../sync'; import type { PullArgv } from './argv'; import { getGoogleGitSyncPrerequisites, parseSelectedSynsets } from './common'; +function toUndefined(value: string): string | undefined { + return value === '' ? undefined : value; +} + export async function pull(argv: PullArgv) { + const [spreadsheetName, mainLanguagesSheet, additionalLanguagesSheet] = argv.source + ? argv.source.split(':').map(toUndefined) + : []; + const { multisynsets, words, wordsAddLang } = - await getGoogleGitSyncPrerequisites(); + await getGoogleGitSyncPrerequisites({ + spreadsheetName, + mainLanguagesSheet, + additionalLanguagesSheet, + }); const selectedIds = await parseSelectedSynsets(multisynsets, argv); if (argv.only && !selectedIds) { diff --git a/packages/engine/src/cli/synsets-cmd/push.ts b/packages/engine/src/cli/synsets-cmd/push.ts index c765d06..09a2519 100644 --- a/packages/engine/src/cli/synsets-cmd/push.ts +++ b/packages/engine/src/cli/synsets-cmd/push.ts @@ -7,6 +7,10 @@ export async function push(argv: PushArgv) { const { words, wordsAddLang, multisynsets } = await getGoogleGitSyncPrerequisites(); + if (!wordsAddLang) { + throw new Error('Cannot find the sheet: words_add_lang'); + } + const selectedIds = await parseSelectedSynsets(multisynsets, argv); if (argv.only && !selectedIds) { console.log( diff --git a/packages/engine/src/cli/synsets.ts b/packages/engine/src/cli/synsets.ts index de12bfe..3f1c926 100644 --- a/packages/engine/src/cli/synsets.ts +++ b/packages/engine/src/cli/synsets.ts @@ -39,6 +39,10 @@ export const builder: CommandBuilder = { description: 'Partial sync (disable deletion)', default: false, }, + source: { + type: 'string', + description: 'Custom source spreadsheet', + }, only: { type: 'boolean', description: 'Only selected synsets', diff --git a/packages/engine/src/sync/words/GSheetsOp.ts b/packages/engine/src/sync/words/GSheetsOp.ts index 78128e8..7a12897 100644 --- a/packages/engine/src/sync/words/GSheetsOp.ts +++ b/packages/engine/src/sync/words/GSheetsOp.ts @@ -13,7 +13,7 @@ import { log } from '../../utils'; export type GSheetsOpOptions = { readonly words: WordsSheet; - readonly wordsAddLang: WordsAddLangSheet; + readonly wordsAddLang?: WordsAddLangSheet; readonly selectedIds?: number[]; readonly multisynsets: MultilingualSynsetRepository; /** Disables deletion of entities */ @@ -25,8 +25,9 @@ type TableDTO = WordsRecord | WordsAddLangRecord; export abstract class GSheetsOp extends IdSyncOperation { private _words?: Promise>; private _wordsAdd?: Promise>; + private _maxId?: Promise; protected readonly wordsSheet: WordsSheet; - protected readonly wordsAddLangSheet: WordsAddLangSheet; + protected readonly wordsAddLangSheet?: WordsAddLangSheet; protected readonly multisynsets: MultilingualSynsetRepository; protected readonly selectedIds?: Set; @@ -49,6 +50,15 @@ export abstract class GSheetsOp extends IdSyncOperation { await this.wordsAdd(); } + protected async maxFsId(): Promise { + if (!this._maxId) { + // eslint-disable-next-line unicorn/no-array-reduce + this._maxId = this.multisynsets.keys().then((keys) => keys.reduce((a, b) => Math.max(Math.abs(a), Math.abs(b)), 0)); + } + + return this._maxId; + } + protected async wordIds(): Promise { return this.words().then((r) => [...r.keys()]); } @@ -63,7 +73,9 @@ export abstract class GSheetsOp extends IdSyncOperation { protected async wordsAdd(): Promise> { if (!this._wordsAdd) { - this._wordsAdd = this._getRecords(this.wordsAddLangSheet); + this._wordsAdd = this.wordsAddLangSheet + ? this._getRecords(this.wordsAddLangSheet) + : Promise.resolve(new Map()); } return this._wordsAdd; @@ -76,10 +88,19 @@ export abstract class GSheetsOp extends IdSyncOperation { { cat: ['gsheets'], tid: ['sync', sheet.id] }, `fetch ${sheet.title}`, async () => { + let maxFsId = await this.maxFsId(); const dtos = (await sheet.getValues()) as unknown as DTO[]; const grecords = new Map( dtos - .map((dto) => [Math.abs(+dto.id), dto]), + .map((dto) => { + let id = Math.abs(+dto.id); + if (Number.isNaN(id)) { + id = ++maxFsId; + dto.id = id.toString(); + } + + return [id, dto]; + }), ); return grecords; diff --git a/packages/engine/src/sync/words/Git2Gsheets.ts b/packages/engine/src/sync/words/Git2Gsheets.ts index 979887b..d4c4117 100644 --- a/packages/engine/src/sync/words/Git2Gsheets.ts +++ b/packages/engine/src/sync/words/Git2Gsheets.ts @@ -61,7 +61,7 @@ export class Git2Gsheets extends GSheetsOp { notes, }); - this.wordsAddLangSheet.batch.appendRows({ + this.wordsAddLangSheet!.batch.appendRows({ values: [[...dtoAddLang]], }); } @@ -79,7 +79,7 @@ export class Git2Gsheets extends GSheetsOp { protected async rollbackTransaction(): Promise { this.wordsSheet.batch.clear(); - this.wordsAddLangSheet.batch.clear(); + this.wordsAddLangSheet!.batch.clear(); } protected async commit(): Promise { @@ -91,7 +91,7 @@ export class Git2Gsheets extends GSheetsOp { 'batch update', async () => { await this.wordsSheet.batch.flush(); - await this.wordsAddLangSheet.batch.flush(); + await this.wordsAddLangSheet!.batch.flush(); }, ); } @@ -139,7 +139,7 @@ export class Git2Gsheets extends GSheetsOp { } private _synset2dtoAddLang(ms: MultilingualSynset): WordsAddLangDTO { - const Mapper = this.wordsAddLangSheet + const Mapper = this.wordsAddLangSheet! .Mapper as ArrayMapper; const isv = ms.synsets.isv!; // TODO: this is a violation of synset vs lemma separation @@ -199,7 +199,7 @@ export class Git2Gsheets extends GSheetsOp { if (!dto.isv.startsWith('!')) { dto.isv = '!' + dto.isv; - this.wordsAddLangSheet.batch.updateRows({ + this.wordsAddLangSheet!.batch.updateRows({ startRowIndex: dto.getIndex() + 1, startColumnIndex: dto.getColumnIndex('isv'), values: [[dto.isv]], @@ -245,13 +245,13 @@ export class Git2Gsheets extends GSheetsOp { if (dtoOld) { if (this._hasChangesWords(dtoOld, dtoNew)) { const startRowIndex = dtoOld.getIndex() + 1; - this.wordsAddLangSheet.batch.updateRows({ + this.wordsAddLangSheet!.batch.updateRows({ startRowIndex, values: [[...dtoNew]], }); } } else { - this.wordsAddLangSheet.batch.appendRows({ + this.wordsAddLangSheet!.batch.appendRows({ values: [[...dtoNew]], }); }