diff --git a/packages/bigquery/src/lib/__fixtures__/.gitignore b/packages/bigquery/src/lib/__fixtures__/.gitignore new file mode 100644 index 0000000..a6c57f5 --- /dev/null +++ b/packages/bigquery/src/lib/__fixtures__/.gitignore @@ -0,0 +1 @@ +*.json diff --git a/packages/bigquery/src/lib/__fixtures__/tiktok-models.ts b/packages/bigquery/src/lib/__fixtures__/tiktok-models.ts new file mode 100644 index 0000000..99a7037 --- /dev/null +++ b/packages/bigquery/src/lib/__fixtures__/tiktok-models.ts @@ -0,0 +1,100 @@ +import { z, AnyZodObject } from 'zod'; +import { convert } from 'zoq'; + +export const BasePageModel = z.object({ + title: z.string(), + url: z.string(), + pathname: z.string(), + nanoid: z.string(), + created_at: z.coerce.date(), +}); + +export type BasePageModel = z.infer; + +export function withBasePageModel( + page: Incoming +) { + return BasePageModel.merge(page); +} + +export const kTikTokPages = { + sitemap: 'sitemap', + topic: 'topic', + account: 'account', + tag: 'tag', +} as const; + +export type TikTokPage = keyof typeof kTikTokPages; + +const TikTokSitemapTopic = z.object({ + topic: z.string(), + url: z.string(), +}); +export type TikTokSitemapTopic = z.infer; + +const TikTokSitemapAccount = z.object({ + name: z.string(), + url: z.string(), +}); +export type TikTokSitemapAccount = z.infer; + +const TikTokSitemapTag = z.object({ + tag: z.string(), + url: z.string(), +}); +export type TikTokSitemapTag = z.infer; + +const TikTokSitemapPageModel = withBasePageModel( + z.object({ + topics: z.array(TikTokSitemapTopic), + accounts: z.array(TikTokSitemapAccount), + tags: z.array(TikTokSitemapTag), + }) +); +export type TikTokSitemapPageModel = z.infer; + +const TikTokVideo = z.object({ + author: z.string(), + content: z.string(), + music: z.object({ title: z.string(), url: z.string() }), + player: z.object({ image_url: z.string(), video_url: z.string() }), + stars: z.string(), + comments: z.string(), + shares: z.string().nullish(), +}); + +export type TikTokVideo = z.infer; + +const TikTokVideoPageModel = withBasePageModel( + z.object({ + data: z.array(TikTokVideo), + }) +); +export type TikTokVideoPageModel = z.infer; + +// export const TikTokPageModels2 = { +// sitemap: TikTokSitemapPageModel, +// topic: TikTokVideoPageModel, +// account: TikTokVideoPageModel, +// tag: TikTokVideoPageModel, +// } as const; + +// export type TikTokModelName = keyof typeof TikTokPageModels; + +const TikTokPageModels = z.object({ + sitemap: TikTokSitemapPageModel, + topic: TikTokVideoPageModel, + account: TikTokVideoPageModel, + tag: TikTokVideoPageModel, +}); + +export type TikTokPageModels = z.infer; + +const TikTokModelNameEnum = TikTokPageModels.keyof(); + +export type TikTokModelName = z.infer; + +export function getBigQuerySchemaByTikTok(name: TikTokModelName) { + // return convert(TikTokPageModels[name] as any); + return convert(TikTokPageModels.shape[name] as any); +} diff --git a/packages/bigquery/src/lib/__tests__/bigquery-store.test.ts b/packages/bigquery/src/lib/__tests__/bigquery-store.test.ts new file mode 100644 index 0000000..dc2d6d1 --- /dev/null +++ b/packages/bigquery/src/lib/__tests__/bigquery-store.test.ts @@ -0,0 +1,26 @@ +import path from 'path'; +import { BigQueryStore } from '../bigquery-store'; +import { getBigQuerySchemaByTikTok } from '../__fixtures__/tiktok-models'; + +const keyFilename = path.resolve( + __dirname, + '../__fixtures__/shopify-cloud-0-55a469e24366.json' +); + +const projectId = 'shopify-cloud-0'; +const datasetId = 'tiktok_appstore'; + +describe('BigQueryStore', () => { + test('create dataset', () => { + const store = new BigQueryStore( + { + projectId, + keyFilename, + }, + datasetId, + getBigQuerySchemaByTikTok + ); + + store.ensureDataset(); + }); +}); diff --git a/packages/bigquery/tsconfig.json b/packages/bigquery/tsconfig.json index f5b8565..19b9eec 100644 --- a/packages/bigquery/tsconfig.json +++ b/packages/bigquery/tsconfig.json @@ -1,13 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "module": "commonjs", - "forceConsistentCasingInFileNames": true, - "strict": true, - "noImplicitOverride": true, - "noPropertyAccessFromIndexSignature": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true + "module": "commonjs" }, "files": [], "include": [], diff --git a/packages/bigquery/tsconfig.lib.json b/packages/bigquery/tsconfig.lib.json index 33eca2c..ca62a03 100644 --- a/packages/bigquery/tsconfig.lib.json +++ b/packages/bigquery/tsconfig.lib.json @@ -1,6 +1,7 @@ { "extends": "./tsconfig.json", "compilerOptions": { + "module": "commonjs", "outDir": "../../dist/out-tsc", "declaration": true, "types": ["node"] diff --git a/packages/bigquery/tsconfig.spec.json b/packages/bigquery/tsconfig.spec.json index 9b2a121..ff08add 100644 --- a/packages/bigquery/tsconfig.spec.json +++ b/packages/bigquery/tsconfig.spec.json @@ -7,8 +7,14 @@ }, "include": [ "jest.config.ts", - "src/**/*.test.ts", - "src/**/*.spec.ts", - "src/**/*.d.ts" + "**/*.test.ts", + "**/*.spec.ts", + "**/*.test.tsx", + "**/*.spec.tsx", + "**/*.test.js", + "**/*.spec.js", + "**/*.test.jsx", + "**/*.spec.jsx", + "**/*.d.ts" ] } diff --git a/tsconfig.base.json b/tsconfig.base.json index ae84e46..816ed43 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -8,24 +8,22 @@ "emitDecoratorMetadata": true, "experimentalDecorators": true, "allowSyntheticDefaultImports": true, - "resolveJsonModule": true, - "allowJs": false, "esModuleInterop": true, - "preserveConstEnums": true, - "preserveWatchOutput": true, "importHelpers": true, - "target": "es2015", + "target": "esnext", "module": "esnext", - "lib": ["es2017", "dom"], + "lib": ["esnext", "dom"], "skipLibCheck": true, "skipDefaultLibCheck": true, + "noImplicitAny": true, "baseUrl": ".", "paths": { "@placeholdersoft/bigquery": ["packages/bigquery/src/index.ts"], "@placeholdersoft/helpers": ["packages/helpers/src/index.ts"], "@placeholdersoft/typed-env": ["packages/typed-env/src/index.ts"], "@placeholdersoft/typed-env-cli": ["packages/typed-env-cli/src/index.ts"] - } + }, + "typeRoots": ["node_modules/@types", "types"] }, "exclude": ["node_modules", "tmp"] }