diff --git a/.talismanrc b/.talismanrc index 3391ac5dd..b0441d15c 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,34 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 97cb862682f7dec430f2079ac686bbfec4fc22c080c8d50cf14c4e2249bb8c8c + - filename: packages/contentstack-export/src/export/modules/environments.ts + checksum: ee579ab17a42580cd74ebfc24dcb5e0500e1f61958abe8cc9890d3144d2a8d3b + - filename: packages/contentstack-export/src/export/modules/workflows.ts + checksum: d62538c6915d14b754c2f3d1d01c71e82361fce19281f41c6b258bbe4225bfc0 + - filename: packages/contentstack-export/src/export/modules/labels.ts + checksum: bc3d0e099ff17157e4329ce41b4fb46440ebb48543e8cfc77bc30514eb325c2e + - filename: packages/contentstack-export/src/commands/cm/stacks/export.ts + checksum: 544a1384e9144c09c687152151e976f4f7cfc642abf796cd83d955fd440c6df5 + - filename: packages/contentstack-export/src/export/modules/locales.ts + checksum: fde0b7d1e9f857f09e3ae9972513ea3886e72e52fcf574e7f0613e204c29c616 + - filename: packages/contentstack-export/src/export/modules/extensions.ts + checksum: 3a130e782b2734822254c24e0ada301261aeb75bcf7338ebd90334e1e200d63f + - filename: packages/contentstack-export/src/utils/basic-login.ts + checksum: 5f654ddc90a4af5e1e65e31f18dcf41897fcd328b096fc38b5b759fb10a6189c + - filename: packages/contentstack-export/src/export/modules/custom-roles.ts + checksum: b392f2ad72f338e54cbf7dcf08290408cb5a7c91087702a429521ea2da48477e + - filename: packages/contentstack-export/src/utils/export-config-handler.ts + checksum: 83e8bef77dfe5171f5549f9e44975508bfb7bba3f8ef110611e995d4783057a8 + - filename: packages/contentstack-export/src/export/modules/webhooks.ts + checksum: 84cef1cb7a949460866465d6dc87ee7de77dd7993c72523f6b93ac4b934e611a + - filename: packages/contentstack-export/src/export/modules/stack.ts + checksum: 74c8222bc09563d6407b792fc33445d6664c50060e84d1be85e50206c565dc25 + - filename: packages/contentstack-export/src/export/modules/entries.ts + checksum: ffb79e14177a8722e0bcad9e95874cac249a17c6408915e10b1687d7a5cdcc1e + - filename: packages/contentstack-export/src/export/modules/assets.ts + checksum: 8d23c92daea085f5e4c40777f1a1b3f95cf1be0b4caf1e16658d0a763fb96e64 + - filename: packages/contentstack-export/src/types/default-config.ts + checksum: cccb5a18f1e191119a636786eb4b70d4f8bfb4cb25c96cd1981d2802ded66932 + - filename: packages/contentstack-export/test/unit/export/modules/publishing-rules.test.ts + checksum: 76068c1ca9d0837b5fd3eb98e297c7df6b89d0c2ca30d476c1eadfc6e95ac723 + - filename: packages/contentstack-export/src/config/index.ts + checksum: 9148f79ef833f6dae4bbb77c3434e41b0b3c7bfa83877837f60e5c2db888f0bf version: '1.0' diff --git a/packages/contentstack-export/src/commands/cm/stacks/export.ts b/packages/contentstack-export/src/commands/cm/stacks/export.ts index 15f88c573..533147484 100644 --- a/packages/contentstack-export/src/commands/cm/stacks/export.ts +++ b/packages/contentstack-export/src/commands/cm/stacks/export.ts @@ -1,19 +1,17 @@ import { Command } from '@contentstack/cli-command'; import { - cliux, - messageHandler, - printFlagDeprecation, - managementSDKClient, - flags, ContentstackClient, FlagInput, + createLogContext, + flags, + getLogPath, + handleAndLogError, + log, + managementSDKClient, + messageHandler, pathValidator, + printFlagDeprecation, sanitizePath, - configHandler, - log, - handleAndLogError, - getLogPath, - createLogContext, } from '@contentstack/cli-utilities'; import { ModuleExporter } from '../../../export'; @@ -21,6 +19,8 @@ import { ExportConfig } from '../../../types'; import { setupExportConfig, writeExportMetaFile } from '../../../utils'; export default class ExportCommand extends Command { + static aliases: string[] = ['cm:export']; + static description: string = messageHandler.parse('Export content from a stack'); static examples: string[] = [ @@ -33,23 +33,39 @@ export default class ExportCommand extends Command { 'csdx cm:stacks:export --branch [optional] branch name', ]; - static usage: string = - 'cm:stacks:export [-c ] [-k ] [-d ] [-a ] [--module ] [--content-types ] [--branch ] [--secured-assets]'; - static flags: FlagInput = { + alias: flags.string({ + char: 'a', + description: 'The management token alias of the source stack from which you will export content.', + }), + 'auth-token': flags.boolean({ + char: 'A', + description: 'to use auth token', + hidden: true, + parse: printFlagDeprecation(['-A', '--auth-token']), + }), + branch: flags.string({ + char: 'B', + // default: 'main', + description: + "[optional] The name of the branch where you want to export your content. If you don't mention the branch name, then by default the content will be exported from all the branches of your stack.", + exclusive: ['branch-alias'], + parse: printFlagDeprecation(['-B'], ['--branch']), + }), + 'branch-alias': flags.string({ + description: '(Optional) The alias of the branch from which you want to export content.', + exclusive: ['branch'], + }), config: flags.string({ char: 'c', description: '[optional] Path of the config', }), - 'stack-uid': flags.string({ - char: 's', - description: 'API key of the source stack', - hidden: true, - parse: printFlagDeprecation(['-s', '--stack-uid'], ['-k', '--stack-api-key']), - }), - 'stack-api-key': flags.string({ - char: 'k', - description: 'API Key of the source stack', + 'content-types': flags.string({ + char: 't', + description: + '[optional] The UID of the content type(s) whose content you want to export. In case of multiple content types, specify the IDs separated by spaces.', + multiple: true, + parse: printFlagDeprecation(['-t'], ['--content-types']), }), data: flags.string({ description: 'path or location to store the data', @@ -60,61 +76,43 @@ export default class ExportCommand extends Command { char: 'd', description: 'The path or the location in your file system to store the exported content. For e.g., ./content', }), - alias: flags.string({ - char: 'a', - description: 'The management token alias of the source stack from which you will export content.', - }), 'management-token-alias': flags.string({ description: 'alias of the management token', hidden: true, parse: printFlagDeprecation(['--management-token-alias'], ['-a', '--alias']), }), - 'auth-token': flags.boolean({ - char: 'A', - description: 'to use auth token', - hidden: true, - parse: printFlagDeprecation(['-A', '--auth-token']), - }), module: flags.string({ char: 'm', description: '[optional] Specific module name. If not specified, the export command will export all the modules to the stack. The available modules are assets, content-types, entries, environments, extensions, marketplace-apps, global-fields, labels, locales, webhooks, workflows, custom-roles, taxonomies, and studio.', parse: printFlagDeprecation(['-m'], ['--module']), }), - 'content-types': flags.string({ - char: 't', - description: - '[optional] The UID of the content type(s) whose content you want to export. In case of multiple content types, specify the IDs separated by spaces.', - multiple: true, - parse: printFlagDeprecation(['-t'], ['--content-types']), - }), - branch: flags.string({ - char: 'B', - // default: 'main', - description: - "[optional] The name of the branch where you want to export your content. If you don't mention the branch name, then by default the content will be exported from all the branches of your stack.", - parse: printFlagDeprecation(['-B'], ['--branch']), - exclusive: ['branch-alias'], - }), - 'branch-alias': flags.string({ - description: '(Optional) The alias of the branch from which you want to export content.', - exclusive: ['branch'], + query: flags.string({ + description: '[optional] Query object (inline JSON or file path) to filter module exports.', + hidden: true, }), 'secured-assets': flags.boolean({ description: '[optional] Use this flag for assets that are secured.', }), + 'stack-api-key': flags.string({ + char: 'k', + description: 'API Key of the source stack', + }), + 'stack-uid': flags.string({ + char: 's', + description: 'API key of the source stack', + hidden: true, + parse: printFlagDeprecation(['-s', '--stack-uid'], ['-k', '--stack-api-key']), + }), yes: flags.boolean({ char: 'y', - required: false, description: '[optional] Force override all Marketplace prompts.', - }), - query: flags.string({ - description: '[optional] Query object (inline JSON or file path) to filter module exports.', - hidden: true, + required: false, }), }; - static aliases: string[] = ['cm:export']; + static usage = + 'cm:stacks:export [-c ] [-k ] [-d ] [-a ] [--module ] [--content-types ] [--branch ] [--secured-assets]'; async run(): Promise { let exportDir: string = pathValidator('logs'); diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 85aa028fa..06b931dc3 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -1,17 +1,30 @@ import { DefaultConfig } from '../types'; const config: DefaultConfig = { + apis: { + assets: '/assets/', + content_types: '/content_types/', + entries: '/entries/', + environments: '/environments/', + extension: '/extensions', + globalfields: '/global_fields/', + labels: '/labels/', + locales: '/locales/', + stacks: '/stacks/', + userSession: '/user-session/', + users: '/stacks', + webhooks: '/webhooks/', + }, contentVersion: 2, - versioning: false, - host: 'https://api.contentstack.io/v3', + developerHubBaseUrl: '', developerHubUrls: { // NOTE CDA url used as developer-hub url mapper to avoid conflict if user used any custom name 'https://api.contentstack.io': 'https://developerhub-api.contentstack.com', - 'https://eu-api.contentstack.com': 'https://eu-developerhub-api.contentstack.com', - 'https://azure-na-api.contentstack.com': 'https://azure-na-developerhub-api.contentstack.com', 'https://azure-eu-api.contentstack.com': 'https://azure-eu-developerhub-api.contentstack.com', - 'https://gcp-na-api.contentstack.com': 'https://gcp-na-developerhub-api.contentstack.com', + 'https://azure-na-api.contentstack.com': 'https://azure-na-developerhub-api.contentstack.com', + 'https://eu-api.contentstack.com': 'https://eu-developerhub-api.contentstack.com', 'https://gcp-eu-api.contentstack.com': 'https://gcp-eu-developerhub-api.contentstack.com', + 'https://gcp-na-api.contentstack.com': 'https://gcp-na-developerhub-api.contentstack.com', }, // use below hosts for eu region // host:'https://eu-api.contentstack.com/v3', @@ -22,113 +35,303 @@ const config: DefaultConfig = { // use below hosts for gcp-na region // host: 'https://gcp-na-api.contentstack.com' // use below hosts for gcp-eu region + fetchConcurrency: 5, + host: 'https://api.contentstack.io/v3', + languagesCode: [ + 'af-za', + 'sq-al', + 'ar', + 'ar-dz', + 'ar-bh', + 'ar-eg', + 'ar-iq', + 'ar-jo', + 'ar-kw', + 'ar-lb', + 'ar-ly', + 'ar-ma', + 'ar-om', + 'ar-qa', + 'ar-sa', + 'ar-sy', + 'ar-tn', + 'ar-ae', + 'ar-ye', + 'hy-am', + 'az', + 'cy-az-az', + 'lt-az-az', + 'eu-es', + 'be-by', + 'bs', + 'bg-bg', + 'ca-es', + 'zh', + 'zh-au', + 'zh-cn', + 'zh-hk', + 'zh-mo', + 'zh-my', + 'zh-sg', + 'zh-tw', + 'zh-chs', + 'zh-cht', + 'hr-hr', + 'cs', + 'cs-cz', + 'da-dk', + 'div-mv', + 'nl', + 'nl-be', + 'nl-nl', + 'en', + 'en-au', + 'en-at', + 'en-be', + 'en-bz', + 'en-ca', + 'en-cb', + 'en-cn', + 'en-cz', + 'en-dk', + 'en-do', + 'en-ee', + 'en-fi', + 'en-fr', + 'en-de', + 'en-gr', + 'en-hk', + 'en-hu', + 'en-in', + 'en-id', + 'en-ie', + 'en-it', + 'en-jm', + 'en-jp', + 'en-kr', + 'en-lv', + 'en-lt', + 'en-lu', + 'en-my', + 'en-mx', + 'en-nz', + 'en-no', + 'en-ph', + 'en-pl', + 'en-pt', + 'en-pr', + 'en-ru', + 'en-sg', + 'en-sk', + 'en-si', + 'en-za', + 'en-es', + 'en-se', + 'en-ch', + 'en-th', + 'en-nl', + 'en-tt', + 'en-gb', + 'en-us', + 'en-zw', + 'et-ee', + 'fo-fo', + 'fa-ir', + 'fi', + 'fi-fi', + 'fr', + 'fr-be', + 'fr-ca', + 'fr-fr', + 'fr-lu', + 'fr-mc', + 'fr-ch', + 'fr-us', + 'gd', + 'gl-es', + 'ka-ge', + 'de', + 'de-at', + 'de-de', + 'de-li', + 'de-lu', + 'de-ch', + 'el-gr', + 'gu-in', + 'he-il', + 'hi-in', + 'hu-hu', + 'is-is', + 'id-id', + 'it', + 'it-it', + 'it-ch', + 'ja', + 'ja-jp', + 'kn-in', + 'kk-kz', + 'km-kh', + 'kok-in', + 'ko', + 'ko-kr', + 'ky-kz', + 'lv-lv', + 'lt-lt', + 'mk-mk', + 'ms', + 'ms-bn', + 'ms-my', + 'ms-sg', + 'mt', + 'mr-in', + 'mn-mn', + 'no', + 'no-no', + 'nb-no', + 'nn-no', + 'pl-pl', + 'pt', + 'pt-br', + 'pt-pt', + 'pa-in', + 'ro-ro', + 'ru', + 'ru-kz', + 'ru-ru', + 'ru-ua', + 'sa-in', + 'cy-sr-sp', + 'lt-sr-sp', + 'sr-me', + 'sk-sk', + 'sl-si', + 'es', + 'es-ar', + 'es-bo', + 'es-cl', + 'es-co', + 'es-cr', + 'es-do', + 'es-ec', + 'es-sv', + 'es-gt', + 'es-hn', + 'es-419', + 'es-mx', + 'es-ni', + 'es-pa', + 'es-py', + 'es-pe', + 'es-pr', + 'es-es', + 'es-us', + 'es-uy', + 'es-ve', + 'sw-ke', + 'sv', + 'sv-fi', + 'sv-se', + 'syr-sy', + 'tl', + 'ta-in', + 'tt-ru', + 'te-in', + 'th-th', + 'tr-tr', + 'uk-ua', + 'ur-pk', + 'uz', + 'cy-uz-uz', + 'lt-uz-uz', + 'vi-vn', + 'xh', + 'zu', + ], + marketplaceAppEncryptionKey: 'nF2ejRQcTv', // host: 'https://gcp-eu-api.contentstack.com' modules: { - types: [ - 'stack', - 'assets', - 'locales', - 'environments', - 'extensions', - 'webhooks', - 'taxonomies', - 'global-fields', - 'content-types', - 'custom-roles', - 'workflows', - 'personalize', - 'entries', - 'labels', - 'marketplace-apps', - 'composable-studio', - ], - locales: { - dirName: 'locales', - fileName: 'locales.json', - requiredKeys: ['code', 'uid', 'name', 'fallback_locale'], - }, - masterLocale: { - dirName: 'locales', - fileName: 'master-locale.json', - requiredKeys: ['code', 'uid', 'name'], - }, - customRoles: { - dirName: 'custom-roles', - fileName: 'custom-roles.json', - customRolesLocalesFileName: 'custom-roles-locales.json', - }, - 'custom-roles': { - dirName: 'custom-roles', - fileName: 'custom-roles.json', - customRolesLocalesFileName: 'custom-roles-locales.json', - }, - environments: { - dirName: 'environments', - fileName: 'environments.json', - }, - labels: { - dirName: 'labels', - fileName: 'labels.json', - invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], - }, - webhooks: { - dirName: 'webhooks', - fileName: 'webhooks.json', - }, - releases: { - dirName: 'releases', - fileName: 'releases.json', - releasesList: 'releasesList.json', - invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], - }, - workflows: { - dirName: 'workflows', - fileName: 'workflows.json', - invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], - }, - globalfields: { - dirName: 'global_fields', - fileName: 'globalfields.json', - validKeys: ['title', 'uid', 'schema', 'options', 'singleton', 'description'], - }, - 'global-fields': { - dirName: 'global_fields', - fileName: 'globalfields.json', - validKeys: ['title', 'uid', 'schema', 'options', 'singleton', 'description'], - }, assets: { - dirName: 'assets', - fileName: 'assets.json', + assetsMetaKeys: [], // Default keys ['uid', 'url', 'filename'] // This is the total no. of asset objects fetched in each 'get assets' call batchLimit: 20, - host: 'https://images.contentstack.io', - invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by', '_metadata', 'published'], // no of asset version files (of a single asset) that'll be downloaded parallel chunkFileSize: 1, // measured on Megabits (5mb) - downloadLimit: 5, - fetchConcurrency: 5, - assetsMetaKeys: [], // Default keys ['uid', 'url', 'filename'] - securedAssets: false, + dirName: 'assets', displayExecutionTime: false, + downloadLimit: 5, enableDownloadStatus: false, + fetchConcurrency: 5, + fileName: 'assets.json', + host: 'https://images.contentstack.io', includeVersionedAssets: false, + invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by', '_metadata', 'published'], + securedAssets: false, + }, + attributes: { + dirName: 'attributes', + fileName: 'attributes.json', + invalidKeys: [ + 'updatedAt', + 'createdBy', + 'updatedBy', + '_id', + 'createdAt', + 'createdByUserName', + 'updatedByUserName', + ], + }, + audiences: { + dirName: 'audiences', + fileName: 'audiences.json', + invalidKeys: [ + 'updatedAt', + 'createdBy', + 'updatedBy', + '_id', + 'createdAt', + 'createdByUserName', + 'updatedByUserName', + ], + }, + 'composable-studio': { + apiBaseUrl: 'https://composable-studio-api.contentstack.com', + apiVersion: 'v1', + dirName: 'composable_studio', + fileName: 'composable_studio.json', }, content_types: { dirName: 'content_types', fileName: 'content_types.json', - validKeys: ['title', 'uid', 'field_rules', 'schema', 'options', 'singleton', 'description'], // total no of content types fetched in each 'get content types' call limit: 100, + validKeys: ['title', 'uid', 'field_rules', 'schema', 'options', 'singleton', 'description'], }, 'content-types': { dirName: 'content_types', fileName: 'content_types.json', - validKeys: ['title', 'uid', 'field_rules', 'schema', 'options', 'singleton', 'description'], // total no of content types fetched in each 'get content types' call limit: 100, + validKeys: ['title', 'uid', 'field_rules', 'schema', 'options', 'singleton', 'description'], + }, + 'custom-roles': { + customRolesLocalesFileName: 'custom-roles-locales.json', + dirName: 'custom-roles', + fileName: 'custom-roles.json', + }, + customRoles: { + customRolesLocalesFileName: 'custom-roles-locales.json', + dirName: 'custom-roles', + fileName: 'custom-roles.json', + }, + dependency: { + entries: ['stack', 'locales', 'content-types'], }, entries: { + batchLimit: 20, + dependencies: ['locales', 'content-types'], dirName: 'entries', + downloadLimit: 5, + exportVersions: false, fileName: 'entries.json', invalidKeys: [ 'stackHeaders', @@ -141,29 +344,64 @@ const config: DefaultConfig = { '_metadata', 'published', ], - batchLimit: 20, - downloadLimit: 5, // total no of entries fetched in each content type in a single call limit: 100, - dependencies: ['locales', 'content-types'], - exportVersions: false, + }, + environments: { + dirName: 'environments', + fileName: 'environments.json', + }, + events: { + dirName: 'events', + fileName: 'events.json', + invalidKeys: [ + 'updatedAt', + 'createdBy', + 'updatedBy', + '_id', + 'createdAt', + 'createdByUserName', + 'updatedByUserName', + ], + }, + extensions: { + dirName: 'extensions', + fileName: 'extensions.json', + }, + 'global-fields': { + dirName: 'global_fields', + fileName: 'globalfields.json', + validKeys: ['title', 'uid', 'schema', 'options', 'singleton', 'description'], + }, + globalfields: { + dirName: 'global_fields', + fileName: 'globalfields.json', + validKeys: ['title', 'uid', 'schema', 'options', 'singleton', 'description'], + }, + labels: { + dirName: 'labels', + fileName: 'labels.json', + invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], + }, + locales: { + dirName: 'locales', + fileName: 'locales.json', + requiredKeys: ['code', 'uid', 'name', 'fallback_locale'], + }, + marketplace_apps: { + dirName: 'marketplace_apps', + fileName: 'marketplace_apps.json', + }, + 'marketplace-apps': { + dirName: 'marketplace_apps', + fileName: 'marketplace_apps.json', + }, + masterLocale: { + dirName: 'locales', + fileName: 'master-locale.json', + requiredKeys: ['code', 'uid', 'name'], }, personalize: { - baseURL: { - 'AWS-NA': 'https://personalize-api.contentstack.com', - 'AWS-EU': 'https://eu-personalize-api.contentstack.com', - 'AWS-AU': 'https://au-personalize-api.contentstack.com', - 'AZURE-NA': 'https://azure-na-personalize-api.contentstack.com', - 'AZURE-EU': 'https://azure-eu-personalize-api.contentstack.com', - 'GCP-NA': 'https://gcp-na-personalize-api.contentstack.com', - 'GCP-EU': 'https://gcp-eu-personalize-api.contentstack.com', - }, - dirName: 'personalize', - exportOrder: ['attributes', 'audiences', 'events', 'experiences'], - projects: { - dirName: 'projects', - fileName: 'projects.json', - }, attributes: { dirName: 'attributes', fileName: 'attributes.json', @@ -172,6 +410,16 @@ const config: DefaultConfig = { dirName: 'audiences', fileName: 'audiences.json', }, + baseURL: { + 'AWS-AU': 'https://au-personalize-api.contentstack.com', + 'AWS-EU': 'https://eu-personalize-api.contentstack.com', + 'AWS-NA': 'https://personalize-api.contentstack.com', + 'AZURE-EU': 'https://azure-eu-personalize-api.contentstack.com', + 'AZURE-NA': 'https://azure-na-personalize-api.contentstack.com', + 'GCP-EU': 'https://gcp-eu-personalize-api.contentstack.com', + 'GCP-NA': 'https://gcp-na-personalize-api.contentstack.com', + }, + dirName: 'personalize', events: { dirName: 'events', fileName: 'events.json', @@ -180,323 +428,81 @@ const config: DefaultConfig = { dirName: 'experiences', fileName: 'experiences.json', }, - }, - variantEntry: { - serveMockData: false, - dirName: 'variants', - fileName: 'index.json', - chunkFileSize: 1, - query: { - skip: 0, - limit: 100, - include_variant: false, - include_count: true, - include_publish_details: true, + exportOrder: ['attributes', 'audiences', 'events', 'experiences'], + projects: { + dirName: 'projects', + fileName: 'projects.json', }, - mockDataPath: './variant-mock-data.json', }, - extensions: { - dirName: 'extensions', - fileName: 'extensions.json', + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], + }, + releases: { + dirName: 'releases', + fileName: 'releases.json', + invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], + releasesList: 'releasesList.json', }, stack: { dirName: 'stack', fileName: 'stack.json', }, - dependency: { - entries: ['stack', 'locales', 'content-types'], - }, - marketplace_apps: { - dirName: 'marketplace_apps', - fileName: 'marketplace_apps.json', - }, - 'marketplace-apps': { - dirName: 'marketplace_apps', - fileName: 'marketplace_apps.json', - }, - 'composable-studio': { - dirName: 'composable_studio', - fileName: 'composable_studio.json', - apiBaseUrl: 'https://composable-studio-api.contentstack.com', - apiVersion: 'v1', - }, taxonomies: { dirName: 'taxonomies', fileName: 'taxonomies.json', invalidKeys: ['updated_at', 'created_by', 'updated_by', 'stackHeaders', 'urlPath', 'created_at'], limit: 100, }, - events: { - dirName: 'events', - fileName: 'events.json', - invalidKeys: [ - 'updatedAt', - 'createdBy', - 'updatedBy', - '_id', - 'createdAt', - 'createdByUserName', - 'updatedByUserName', - ], + types: [ + 'stack', + 'assets', + 'locales', + 'environments', + 'extensions', + 'webhooks', + 'taxonomies', + 'global-fields', + 'content-types', + 'custom-roles', + 'workflows', + 'publishing-rules', + 'personalize', + 'entries', + 'labels', + 'marketplace-apps', + 'composable-studio', + ], + variantEntry: { + chunkFileSize: 1, + dirName: 'variants', + fileName: 'index.json', + mockDataPath: './variant-mock-data.json', + query: { + include_count: true, + include_publish_details: true, + include_variant: false, + limit: 100, + skip: 0, + }, + serveMockData: false, }, - audiences: { - dirName: 'audiences', - fileName: 'audiences.json', - invalidKeys: [ - 'updatedAt', - 'createdBy', - 'updatedBy', - '_id', - 'createdAt', - 'createdByUserName', - 'updatedByUserName', - ], + webhooks: { + dirName: 'webhooks', + fileName: 'webhooks.json', }, - attributes: { - dirName: 'attributes', - fileName: 'attributes.json', - invalidKeys: [ - 'updatedAt', - 'createdBy', - 'updatedBy', - '_id', - 'createdAt', - 'createdByUserName', - 'updatedByUserName', - ], + workflows: { + dirName: 'workflows', + fileName: 'workflows.json', + invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], }, }, - languagesCode: [ - 'af-za', - 'sq-al', - 'ar', - 'ar-dz', - 'ar-bh', - 'ar-eg', - 'ar-iq', - 'ar-jo', - 'ar-kw', - 'ar-lb', - 'ar-ly', - 'ar-ma', - 'ar-om', - 'ar-qa', - 'ar-sa', - 'ar-sy', - 'ar-tn', - 'ar-ae', - 'ar-ye', - 'hy-am', - 'az', - 'cy-az-az', - 'lt-az-az', - 'eu-es', - 'be-by', - 'bs', - 'bg-bg', - 'ca-es', - 'zh', - 'zh-au', - 'zh-cn', - 'zh-hk', - 'zh-mo', - 'zh-my', - 'zh-sg', - 'zh-tw', - 'zh-chs', - 'zh-cht', - 'hr-hr', - 'cs', - 'cs-cz', - 'da-dk', - 'div-mv', - 'nl', - 'nl-be', - 'nl-nl', - 'en', - 'en-au', - 'en-at', - 'en-be', - 'en-bz', - 'en-ca', - 'en-cb', - 'en-cn', - 'en-cz', - 'en-dk', - 'en-do', - 'en-ee', - 'en-fi', - 'en-fr', - 'en-de', - 'en-gr', - 'en-hk', - 'en-hu', - 'en-in', - 'en-id', - 'en-ie', - 'en-it', - 'en-jm', - 'en-jp', - 'en-kr', - 'en-lv', - 'en-lt', - 'en-lu', - 'en-my', - 'en-mx', - 'en-nz', - 'en-no', - 'en-ph', - 'en-pl', - 'en-pt', - 'en-pr', - 'en-ru', - 'en-sg', - 'en-sk', - 'en-si', - 'en-za', - 'en-es', - 'en-se', - 'en-ch', - 'en-th', - 'en-nl', - 'en-tt', - 'en-gb', - 'en-us', - 'en-zw', - 'et-ee', - 'fo-fo', - 'fa-ir', - 'fi', - 'fi-fi', - 'fr', - 'fr-be', - 'fr-ca', - 'fr-fr', - 'fr-lu', - 'fr-mc', - 'fr-ch', - 'fr-us', - 'gd', - 'gl-es', - 'ka-ge', - 'de', - 'de-at', - 'de-de', - 'de-li', - 'de-lu', - 'de-ch', - 'el-gr', - 'gu-in', - 'he-il', - 'hi-in', - 'hu-hu', - 'is-is', - 'id-id', - 'it', - 'it-it', - 'it-ch', - 'ja', - 'ja-jp', - 'kn-in', - 'kk-kz', - 'km-kh', - 'kok-in', - 'ko', - 'ko-kr', - 'ky-kz', - 'lv-lv', - 'lt-lt', - 'mk-mk', - 'ms', - 'ms-bn', - 'ms-my', - 'ms-sg', - 'mt', - 'mr-in', - 'mn-mn', - 'no', - 'no-no', - 'nb-no', - 'nn-no', - 'pl-pl', - 'pt', - 'pt-br', - 'pt-pt', - 'pa-in', - 'ro-ro', - 'ru', - 'ru-kz', - 'ru-ru', - 'ru-ua', - 'sa-in', - 'cy-sr-sp', - 'lt-sr-sp', - 'sr-me', - 'sk-sk', - 'sl-si', - 'es', - 'es-ar', - 'es-bo', - 'es-cl', - 'es-co', - 'es-cr', - 'es-do', - 'es-ec', - 'es-sv', - 'es-gt', - 'es-hn', - 'es-419', - 'es-mx', - 'es-ni', - 'es-pa', - 'es-py', - 'es-pe', - 'es-pr', - 'es-es', - 'es-us', - 'es-uy', - 'es-ve', - 'sw-ke', - 'sv', - 'sv-fi', - 'sv-se', - 'syr-sy', - 'tl', - 'ta-in', - 'tt-ru', - 'te-in', - 'th-th', - 'tr-tr', - 'uk-ua', - 'ur-pk', - 'uz', - 'cy-uz-uz', - 'lt-uz-uz', - 'vi-vn', - 'xh', - 'zu', - ], - apis: { - userSession: '/user-session/', - globalfields: '/global_fields/', - locales: '/locales/', - labels: '/labels/', - environments: '/environments/', - assets: '/assets/', - content_types: '/content_types/', - entries: '/entries/', - users: '/stacks', - extension: '/extensions', - webhooks: '/webhooks/', - stacks: '/stacks/', - }, - preserveStackVersion: false, + onlyTSModules: ['taxonomies'], personalizationEnabled: false, - fetchConcurrency: 5, + preserveStackVersion: false, + versioning: false, writeConcurrency: 5, - developerHubBaseUrl: '', - marketplaceAppEncryptionKey: 'nF2ejRQcTv', - onlyTSModules: ['taxonomies'], }; export default config; diff --git a/packages/contentstack-export/src/export/module-exporter.ts b/packages/contentstack-export/src/export/module-exporter.ts index 0cd1a8bea..02077334a 100644 --- a/packages/contentstack-export/src/export/module-exporter.ts +++ b/packages/contentstack-export/src/export/module-exporter.ts @@ -1,19 +1,22 @@ -import * as path from 'path'; import { ContentstackClient, + getBranchFromAlias, handleAndLogError, - messageHandler, log, - getBranchFromAlias, + messageHandler, } from '@contentstack/cli-utilities'; +import * as path from 'path'; + +import { ExportConfig, Modules } from '../types'; import { setupBranches, setupExportDir, writeExportMetaFile } from '../utils'; import startModuleExport from './modules'; +// CJS: modules-js/index.js uses module.exports (no ESM default) +// eslint-disable-next-line import/default -- resolved at runtime via module.exports import startJSModuleExport from './modules-js'; -import { ExportConfig, Modules } from '../types'; class ModuleExporter { - private managementAPIClient: ContentstackClient; private exportConfig: ExportConfig; + private managementAPIClient: ContentstackClient; private stackAPIClient: ReturnType; constructor(managementAPIClient: ContentstackClient, exportConfig: ExportConfig) { @@ -25,22 +28,19 @@ class ModuleExporter { this.exportConfig = exportConfig; } - async start(): Promise { - // setup the branches - try { - if (!this.exportConfig.branchName && this.exportConfig.branchAlias) { - this.exportConfig.branchName = await getBranchFromAlias(this.stackAPIClient, this.exportConfig.branchAlias); - } - await setupBranches(this.exportConfig, this.stackAPIClient); - await setupExportDir(this.exportConfig); - // if branches available run it export by branches - if (this.exportConfig.branches) { - this.exportConfig.branchEnabled = true; - return this.exportByBranches(); - } - return this.export(); - } catch (error) { - throw error; + async export() { + log.info(`Started to export content, version is ${this.exportConfig.contentVersion}`, this.exportConfig.context); + // checks for single module or all modules + if (this.exportConfig.singleModuleExport) { + return this.exportSingleModule(this.exportConfig.moduleName); + } + return this.exportAllModules(); + } + + async exportAllModules(): Promise { + // use the algorithm to determine the parallel and sequential execution of modules + for (const moduleName of this.exportConfig.modules.types) { + await this.exportByModuleByName(moduleName); } } @@ -66,15 +66,6 @@ class ModuleExporter { } } - async export() { - log.info(`Started to export content, version is ${this.exportConfig.contentVersion}`, this.exportConfig.context); - // checks for single module or all modules - if (this.exportConfig.singleModuleExport) { - return this.exportSingleModule(this.exportConfig.moduleName); - } - return this.exportAllModules(); - } - async exportByModuleByName(moduleName: Modules) { log.info(`Exporting module: '${moduleName}'...`, this.exportConfig.context); // export the modules by name @@ -82,17 +73,17 @@ class ModuleExporter { let exportedModuleResponse; if (this.exportConfig.contentVersion === 2) { exportedModuleResponse = await startModuleExport({ - stackAPIClient: this.stackAPIClient, exportConfig: this.exportConfig, moduleName, + stackAPIClient: this.stackAPIClient, }); } else { //NOTE - new modules support only ts if (this.exportConfig.onlyTSModules.indexOf(moduleName) === -1) { exportedModuleResponse = await startJSModuleExport({ - stackAPIClient: this.stackAPIClient, exportConfig: this.exportConfig, moduleName, + stackAPIClient: this.stackAPIClient, }); } } @@ -126,10 +117,22 @@ class ModuleExporter { } } - async exportAllModules(): Promise { - // use the algorithm to determine the parallel and sequential execution of modules - for (const moduleName of this.exportConfig.modules.types) { - await this.exportByModuleByName(moduleName); + async start(): Promise { + // setup the branches + try { + if (!this.exportConfig.branchName && this.exportConfig.branchAlias) { + this.exportConfig.branchName = await getBranchFromAlias(this.stackAPIClient, this.exportConfig.branchAlias); + } + await setupBranches(this.exportConfig, this.stackAPIClient); + await setupExportDir(this.exportConfig); + // if branches available run it export by branches + if (this.exportConfig.branches) { + this.exportConfig.branchEnabled = true; + return this.exportByBranches(); + } + return this.export(); + } catch (error) { + throw error; } } } diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 1a95d5f25..35b193f0e 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -1,34 +1,34 @@ -import map from 'lodash/map'; +import { + FsUtility, + configHandler, + getDirectories, + handleAndLogError, + log, + messageHandler, +} from '@contentstack/cli-utilities'; import chunk from 'lodash/chunk'; +import entries from 'lodash/entries'; +import filter from 'lodash/filter'; import first from 'lodash/first'; +import includes from 'lodash/includes'; +import isEmpty from 'lodash/isEmpty'; +import map from 'lodash/map'; import merge from 'lodash/merge'; -import filter from 'lodash/filter'; import uniqBy from 'lodash/uniqBy'; import values from 'lodash/values'; -import entries from 'lodash/entries'; -import isEmpty from 'lodash/isEmpty'; -import includes from 'lodash/includes'; -import progress from 'progress-stream'; import { createWriteStream } from 'node:fs'; import { resolve as pResolve } from 'node:path'; -import { - FsUtility, - getDirectories, - configHandler, - log, - handleAndLogError, - messageHandler, -} from '@contentstack/cli-utilities'; +import progress from 'progress-stream'; import config from '../../config'; import { ModuleClassParams } from '../../types'; import BaseClass, { CustomPromiseHandler, CustomPromiseHandlerInput } from './base-class'; export default class ExportAssets extends BaseClass { - private assetsRootPath: string; public assetConfig = config.modules.assets; - private assetsFolder: Record[] = []; public versionedAssets: Record[] = []; + private assetsFolder: Record[] = []; + private assetsRootPath: string; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -37,85 +37,120 @@ export default class ExportAssets extends BaseClass { get commonQueryParam(): Record { return { - skip: 0, asc: 'created_at', include_count: false, + skip: 0, }; } - async start(): Promise { - this.assetsRootPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.assetConfig.dirName, - ); + /** + * @method downloadAssets + * @returns Promise + */ + async downloadAssets(): Promise { + const fs: FsUtility = new FsUtility({ + basePath: this.assetsRootPath, + createDirIfNotExist: false, + fileExt: 'json', + }); - log.debug(`Assets root path resolved to: ${this.assetsRootPath}`, this.exportConfig.context); - log.debug('Fetching assets and folders count...', this.exportConfig.context); - // NOTE step 1: Get assets and it's folder count in parallel - const [assetsCount, assetsFolderCount] = await Promise.all([this.getAssetsCount(), this.getAssetsCount(true)]); + log.debug('Reading asset metadata for download...', this.exportConfig.context); + const assetsMetaData = fs.getPlainMeta(); - log.debug('Fetching assets and folders data...', this.exportConfig.context); - // NOTE step 2: Get assets and it's folder data in parallel - await Promise.all([this.getAssetsFolders(assetsFolderCount), this.getAssets(assetsCount)]); + let listOfAssets = values(assetsMetaData).flat(); - // NOTE step 3: Get versioned assets - if (!isEmpty(this.versionedAssets) && this.assetConfig.includeVersionedAssets) { - log.debug('Fetching versioned assets metadata...', this.exportConfig.context); - await this.getVersionedAssets(); + if (this.assetConfig.includeVersionedAssets) { + const versionedAssetsMetaData = fs.getPlainMeta(pResolve(this.assetsRootPath, 'versions', 'metadata.json')); + listOfAssets.push(...values(versionedAssetsMetaData).flat()); } - log.debug('Starting download of all assets...', this.exportConfig.context); - // NOTE step 4: Download all assets - await this.downloadAssets(); + listOfAssets = uniqBy(listOfAssets, 'url'); + log.debug(`Total unique assets to download: ${listOfAssets.length}`, this.exportConfig.context); - log.success(messageHandler.parse('ASSET_EXPORT_COMPLETE'), this.exportConfig.context); - } + const apiBatches: Array = chunk(listOfAssets, this.assetConfig.downloadLimit); + const downloadedAssetsDirs = await getDirectories(pResolve(this.assetsRootPath, 'files')); - /** - * @method getAssetsFolders - * @param {number} totalCount number - * @returns Promise - */ - getAssetsFolders(totalCount: number | void): Promise | void> { - if (!totalCount) return Promise.resolve(); + const onSuccess = ({ additionalInfo, response: { data } }: any) => { + const { asset } = additionalInfo; + const assetFolderPath = pResolve(this.assetsRootPath, 'files', asset.uid); + const assetFilePath = pResolve(assetFolderPath, asset.filename); - const queryParam = { - ...this.commonQueryParam, - query: { is_dir: true }, - }; + log.debug(`Saving asset to: ${assetFilePath}`, this.exportConfig.context); - log.debug(`Fetching asset folders with query: ${JSON.stringify(queryParam)}`, this.exportConfig.context); + if (!includes(downloadedAssetsDirs, asset.uid)) { + fs.createFolderIfNotExist(assetFolderPath); + } - const onSuccess = ({ response: { items } }: any) => { - log.debug(`Fetched ${items?.length || 0} asset folders`, this.exportConfig.context); - if (!isEmpty(items)) this.assetsFolder.push(...items); + const assetWriterStream = createWriteStream(assetFilePath); + assetWriterStream.on('error', (error) => { + handleAndLogError( + error, + { ...this.exportConfig.context, filename: asset.fileName, uid: asset.uid }, + messageHandler.parse('ASSET_DOWNLOAD_FAILED', asset.filename, asset.uid), + ); + }); + /** + * NOTE if pipe not working as expected add the following code below to fix the issue + * https://oramind.com/using-streams-efficiently-in-nodejs/ + * import * as stream from "stream"; + * import { promisify } from "util"; + * const finished = promisify(stream.finished); + * await finished(assetWriterStream); + */ + if (this.assetConfig.enableDownloadStatus) { + const str = progress({ + length: data.headers['content-length'], + time: 5000, + }); + str.on('progress', function (progressData) { + console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`); + }); + data.pipe(str).pipe(assetWriterStream); + } else { + data.pipe(assetWriterStream); + } + + log.success(messageHandler.parse('ASSET_DOWNLOAD_SUCCESS', asset.filename, asset.uid), this.exportConfig.context); }; - const onReject = ({ error }: any) => { - handleAndLogError(error, { ...this.exportConfig.context }); + const onReject = ({ additionalInfo, error }: any) => { + const { asset } = additionalInfo; + handleAndLogError( + error, + { ...this.exportConfig.context, filename: asset.filename, uid: asset.uid }, + messageHandler.parse('ASSET_DOWNLOAD_FAILED', asset.filename, asset.uid), + ); }; - return this.makeConcurrentCall({ - totalCount, - apiParams: { - queryParam, - module: 'assets', + const promisifyHandler: CustomPromiseHandler = (input: CustomPromiseHandlerInput) => { + const { batchIndex, index } = input; + const asset: any = apiBatches[batchIndex][index]; + const url = this.assetConfig.securedAssets + ? `${asset.url}?authtoken=${configHandler.get('authtoken')}` + : asset.url; + log.debug( + `Preparing to download asset: ${asset.filename} (UID: ${asset.uid}) from URL: ${url}`, + this.exportConfig.context, + ); + return this.makeAPICall({ + additionalInfo: { asset }, + module: 'download-asset', reject: onReject, resolve: onSuccess, + url: encodeURI(url), + }); + }; + + return this.makeConcurrentCall( + { + apiBatches, + concurrencyLimit: this.assetConfig.downloadLimit, + module: 'assets download', + totalCount: listOfAssets.length, }, - module: 'assets folders', - concurrencyLimit: this.assetConfig.fetchConcurrency, - }).then(() => { - if (!isEmpty(this.assetsFolder)) { - const path = pResolve(this.assetsRootPath, 'folders.json'); - log.debug(`Writing asset folders to ${path}`, this.exportConfig.context); - new FsUtility({ basePath: this.assetsRootPath }).writeFile(path, this.assetsFolder); - } - log.info( - messageHandler.parse('ASSET_FOLDERS_EXPORT_COMPLETE', this.assetsFolder.length), - this.exportConfig.context, - ); + promisifyHandler, + ).then(() => { + log.success(messageHandler.parse('ASSET_DOWNLOAD_COMPLETE'), this.exportConfig.context); }); } @@ -134,8 +169,8 @@ export default class ExportAssets extends BaseClass { const queryParam = { ...this.commonQueryParam, - include_publish_details: true, except: { BASE: this.assetConfig.invalidKeys }, + include_publish_details: true, }; this.applyQueryFilters(queryParam, 'assets'); @@ -145,7 +180,7 @@ export default class ExportAssets extends BaseClass { log.debug(`Found ${versionAssets.length} versioned assets`, this.exportConfig.context); if (!isEmpty(versionAssets)) { this.versionedAssets.push( - ...map(versionAssets, ({ uid, _version }: any) => ({ + ...map(versionAssets, ({ _version, uid }: any) => ({ [uid]: _version, })), ); @@ -163,12 +198,12 @@ export default class ExportAssets extends BaseClass { if (!fs && !isEmpty(items)) { log.debug('Initializing FsUtility for writing assets metadata', this.exportConfig.context); fs = new FsUtility({ - metaHandler, - moduleName: 'assets', - indexFileName: 'assets.json', basePath: this.assetsRootPath, chunkFileSize: this.assetConfig.chunkFileSize, + indexFileName: 'assets.json', + metaHandler, metaPickKeys: merge(['uid', 'url', 'filename', 'parent_uid'], this.assetConfig.assetsMetaKeys), + moduleName: 'assets', }); } if (!isEmpty(items)) { @@ -178,20 +213,94 @@ export default class ExportAssets extends BaseClass { }; return this.makeConcurrentCall({ - module: 'assets', - totalCount, apiParams: { - queryParam, module: 'assets', + queryParam, reject: onReject, resolve: onSuccess, }, concurrencyLimit: this.assetConfig.fetchConcurrency, + module: 'assets', + totalCount, }).then(() => { fs?.completeFile(true); log.info(messageHandler.parse('ASSET_METADATA_EXPORT_COMPLETE'), this.exportConfig.context); }); } + + getAssetsCount(isDir = false): Promise { + const queryParam: any = { + limit: 1, + ...this.commonQueryParam, + skip: 10 ** 100, + }; + + if (isDir) queryParam.query = { is_dir: true }; + + log.debug( + `Querying count of assets${isDir ? ' (folders only)' : ''} with params: ${JSON.stringify(queryParam)}`, + this.exportConfig.context, + ); + + return this.stack + .asset() + .query(queryParam) + .count() + .then(({ assets }: any) => { + log.debug(`Received asset count: ${assets}`, this.exportConfig.context); + return assets; + }) + .catch((error: Error) => { + handleAndLogError(error, { ...this.exportConfig.context }, messageHandler.parse('ASSET_COUNT_QUERY_FAILED')); + }); + } + /** + * @method getAssetsFolders + * @param {number} totalCount number + * @returns Promise + */ + getAssetsFolders(totalCount: number | void): Promise | void> { + if (!totalCount) return Promise.resolve(); + + const queryParam = { + ...this.commonQueryParam, + query: { is_dir: true }, + }; + + log.debug(`Fetching asset folders with query: ${JSON.stringify(queryParam)}`, this.exportConfig.context); + + const onSuccess = ({ response: { items } }: any) => { + log.debug(`Fetched ${items?.length || 0} asset folders`, this.exportConfig.context); + if (!isEmpty(items)) this.assetsFolder.push(...items); + }; + + const onReject = ({ error }: any) => { + handleAndLogError(error, { ...this.exportConfig.context }); + }; + + return this.makeConcurrentCall({ + apiParams: { + module: 'assets', + queryParam, + reject: onReject, + resolve: onSuccess, + }, + concurrencyLimit: this.assetConfig.fetchConcurrency, + module: 'assets folders', + totalCount, + }).then(() => { + if (!isEmpty(this.assetsFolder)) { + const path = pResolve(this.assetsRootPath, 'folders.json'); + log.debug(`Writing asset folders to ${path}`, this.exportConfig.context); + new FsUtility({ basePath: this.assetsRootPath }).writeFile(path, this.assetsFolder); + } + log.info( + messageHandler.parse('ASSET_FOLDERS_EXPORT_COMPLETE', this.assetsFolder.length), + this.exportConfig.context, + ); + }); + } + /** * @method getVersionedAssets * @returns Promise @@ -203,8 +312,8 @@ export default class ExportAssets extends BaseClass { const queryParam = { ...this.commonQueryParam, - include_publish_details: true, except: { BASE: this.assetConfig.invalidKeys }, + include_publish_details: true, }; const versionedAssets = map(this.versionedAssets, (element) => { @@ -222,7 +331,7 @@ export default class ExportAssets extends BaseClass { const apiBatches: Array = chunk(versionedAssets, this.assetConfig.fetchConcurrency); const promisifyHandler: CustomPromiseHandler = (input: CustomPromiseHandlerInput) => { - const { index, batchIndex, apiParams, isLastRequest } = input; + const { apiParams, batchIndex, index, isLastRequest } = input; const batch: Record = apiBatches[batchIndex][index]; const [uid, version]: any = first(entries(batch)); @@ -239,11 +348,11 @@ export default class ExportAssets extends BaseClass { const onSuccess = ({ response }: any) => { if (!fs && !isEmpty(response)) { fs = new FsUtility({ - moduleName: 'assets', - indexFileName: 'versioned-assets.json', - chunkFileSize: this.assetConfig.chunkFileSize, basePath: pResolve(this.assetsRootPath, 'versions'), + chunkFileSize: this.assetConfig.chunkFileSize, + indexFileName: 'versioned-assets.json', metaPickKeys: merge(['uid', 'url', 'filename', '_version', 'parent_uid'], this.assetConfig.assetsMetaKeys), + moduleName: 'assets', }); } if (!isEmpty(response)) { @@ -251,7 +360,7 @@ export default class ExportAssets extends BaseClass { `Writing versioned asset: UID=${response.uid}, Version=${response._version}`, this.exportConfig.context, ); - fs?.writeIntoFile([response], { mapKeyVal: true, keyName: ['uid', '_version'] }); + fs?.writeIntoFile([response], { keyName: ['uid', '_version'], mapKeyVal: true }); } }; @@ -263,14 +372,14 @@ export default class ExportAssets extends BaseClass { { apiBatches, apiParams: { - queryParam, module: 'asset', + queryParam, reject: onReject, resolve: onSuccess, }, + concurrencyLimit: this.assetConfig.fetchConcurrency, module: 'versioned assets', totalCount: versionedAssets.length, - concurrencyLimit: this.assetConfig.fetchConcurrency, }, promisifyHandler, ).then(() => { @@ -278,141 +387,32 @@ export default class ExportAssets extends BaseClass { log.info(messageHandler.parse('ASSET_VERSIONED_METADATA_EXPORT_COMPLETE'), this.exportConfig.context); }); } - - getAssetsCount(isDir = false): Promise { - const queryParam: any = { - limit: 1, - ...this.commonQueryParam, - skip: 10 ** 100, - }; - - if (isDir) queryParam.query = { is_dir: true }; - - log.debug( - `Querying count of assets${isDir ? ' (folders only)' : ''} with params: ${JSON.stringify(queryParam)}`, - this.exportConfig.context, + async start(): Promise { + this.assetsRootPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.assetConfig.dirName, ); - return this.stack - .asset() - .query(queryParam) - .count() - .then(({ assets }: any) => { - log.debug(`Received asset count: ${assets}`, this.exportConfig.context); - return assets; - }) - .catch((error: Error) => { - handleAndLogError(error, { ...this.exportConfig.context }, messageHandler.parse('ASSET_COUNT_QUERY_FAILED')); - }); - } - /** - * @method downloadAssets - * @returns Promise - */ - async downloadAssets(): Promise { - const fs: FsUtility = new FsUtility({ - fileExt: 'json', - createDirIfNotExist: false, - basePath: this.assetsRootPath, - }); - - log.debug('Reading asset metadata for download...', this.exportConfig.context); - const assetsMetaData = fs.getPlainMeta(); + log.debug(`Assets root path resolved to: ${this.assetsRootPath}`, this.exportConfig.context); + log.debug('Fetching assets and folders count...', this.exportConfig.context); + // NOTE step 1: Get assets and it's folder count in parallel + const [assetsCount, assetsFolderCount] = await Promise.all([this.getAssetsCount(), this.getAssetsCount(true)]); - let listOfAssets = values(assetsMetaData).flat(); + log.debug('Fetching assets and folders data...', this.exportConfig.context); + // NOTE step 2: Get assets and it's folder data in parallel + await Promise.all([this.getAssetsFolders(assetsFolderCount), this.getAssets(assetsCount)]); - if (this.assetConfig.includeVersionedAssets) { - const versionedAssetsMetaData = fs.getPlainMeta(pResolve(this.assetsRootPath, 'versions', 'metadata.json')); - listOfAssets.push(...values(versionedAssetsMetaData).flat()); + // NOTE step 3: Get versioned assets + if (!isEmpty(this.versionedAssets) && this.assetConfig.includeVersionedAssets) { + log.debug('Fetching versioned assets metadata...', this.exportConfig.context); + await this.getVersionedAssets(); } - listOfAssets = uniqBy(listOfAssets, 'url'); - log.debug(`Total unique assets to download: ${listOfAssets.length}`, this.exportConfig.context); - - const apiBatches: Array = chunk(listOfAssets, this.assetConfig.downloadLimit); - const downloadedAssetsDirs = await getDirectories(pResolve(this.assetsRootPath, 'files')); - - const onSuccess = ({ response: { data }, additionalInfo }: any) => { - const { asset } = additionalInfo; - const assetFolderPath = pResolve(this.assetsRootPath, 'files', asset.uid); - const assetFilePath = pResolve(assetFolderPath, asset.filename); - - log.debug(`Saving asset to: ${assetFilePath}`, this.exportConfig.context); - - if (!includes(downloadedAssetsDirs, asset.uid)) { - fs.createFolderIfNotExist(assetFolderPath); - } - - const assetWriterStream = createWriteStream(assetFilePath); - assetWriterStream.on('error', (error) => { - handleAndLogError( - error, - { ...this.exportConfig.context, uid: asset.uid, filename: asset.fileName }, - messageHandler.parse('ASSET_DOWNLOAD_FAILED', asset.filename, asset.uid), - ); - }); - /** - * NOTE if pipe not working as expected add the following code below to fix the issue - * https://oramind.com/using-streams-efficiently-in-nodejs/ - * import * as stream from "stream"; - * import { promisify } from "util"; - * const finished = promisify(stream.finished); - * await finished(assetWriterStream); - */ - if (this.assetConfig.enableDownloadStatus) { - const str = progress({ - time: 5000, - length: data.headers['content-length'], - }); - str.on('progress', function (progressData) { - console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`); - }); - data.pipe(str).pipe(assetWriterStream); - } else { - data.pipe(assetWriterStream); - } - - log.success(messageHandler.parse('ASSET_DOWNLOAD_SUCCESS', asset.filename, asset.uid), this.exportConfig.context); - }; - - const onReject = ({ error, additionalInfo }: any) => { - const { asset } = additionalInfo; - handleAndLogError( - error, - { ...this.exportConfig.context, uid: asset.uid, filename: asset.filename }, - messageHandler.parse('ASSET_DOWNLOAD_FAILED', asset.filename, asset.uid), - ); - }; - - const promisifyHandler: CustomPromiseHandler = (input: CustomPromiseHandlerInput) => { - const { index, batchIndex } = input; - const asset: any = apiBatches[batchIndex][index]; - const url = this.assetConfig.securedAssets - ? `${asset.url}?authtoken=${configHandler.get('authtoken')}` - : asset.url; - log.debug( - `Preparing to download asset: ${asset.filename} (UID: ${asset.uid}) from URL: ${url}`, - this.exportConfig.context, - ); - return this.makeAPICall({ - reject: onReject, - resolve: onSuccess, - url: encodeURI(url), - module: 'download-asset', - additionalInfo: { asset }, - }); - }; + log.debug('Starting download of all assets...', this.exportConfig.context); + // NOTE step 4: Download all assets + await this.downloadAssets(); - return this.makeConcurrentCall( - { - apiBatches, - module: 'assets download', - totalCount: listOfAssets.length, - concurrencyLimit: this.assetConfig.downloadLimit, - }, - promisifyHandler, - ).then(() => { - log.success(messageHandler.parse('ASSET_DOWNLOAD_COMPLETE'), this.exportConfig.context); - }); + log.success(messageHandler.parse('ASSET_EXPORT_COMPLETE'), this.exportConfig.context); } } diff --git a/packages/contentstack-export/src/export/modules/base-class.ts b/packages/contentstack-export/src/export/modules/base-class.ts index 6379669e1..1a2fd9dda 100644 --- a/packages/contentstack-export/src/export/modules/base-class.ts +++ b/packages/contentstack-export/src/export/modules/base-class.ts @@ -1,54 +1,54 @@ -import map from 'lodash/map'; -import fill from 'lodash/fill'; -import last from 'lodash/last'; +import { log } from '@contentstack/cli-utilities'; import chunk from 'lodash/chunk'; -import isEmpty from 'lodash/isEmpty'; import entries from 'lodash/entries'; +import fill from 'lodash/fill'; +import isEmpty from 'lodash/isEmpty'; import isEqual from 'lodash/isEqual'; -import { log } from '@contentstack/cli-utilities'; +import last from 'lodash/last'; +import map from 'lodash/map'; import { ExportConfig, ModuleClassParams } from '../../types'; export type ApiOptions = { - uid?: string; - url?: string; + additionalInfo?: Record; module: ApiModuleType; queryParam?: Record; - resolve: (value: any) => void; reject: (error: any) => void; - additionalInfo?: Record; + resolve: (value: any) => void; + uid?: string; + url?: string; }; export type EnvType = { - module: string; - totalCount: number; apiBatches?: number[]; - concurrencyLimit: number; apiParams?: ApiOptions; + concurrencyLimit: number; + module: string; + totalCount: number; }; export type CustomPromiseHandlerInput = { - index: number; + apiParams?: ApiOptions; batchIndex: number; element?: Record; - apiParams?: ApiOptions; + index: number; isLastRequest: boolean; }; export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise; export type ApiModuleType = - | 'stack' | 'asset' | 'assets' - | 'entry' - | 'entries' | 'content-type' | 'content-types' - | 'stacks' - | 'versioned-entries' | 'download-asset' - | 'export-taxonomy'; + | 'entries' + | 'entry' + | 'export-taxonomy' + | 'stack' + | 'stacks' + | 'versioned-entries'; export default abstract class BaseClass { readonly client: any; @@ -63,67 +63,25 @@ export default abstract class BaseClass { return this.client; } + protected applyQueryFilters(requestObject: any, moduleName: string): any { + if (this.exportConfig.query?.modules?.[moduleName]) { + const moduleQuery = this.exportConfig.query.modules[moduleName]; + // Merge the query parameters with existing requestObject + if (moduleQuery) { + if (!requestObject.query) { + requestObject.query = moduleQuery; + } + Object.assign(requestObject.query, moduleQuery); + } + } + return requestObject; + } + delay(ms: number): Promise { /* eslint-disable no-promise-executor-return */ return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms)); } - makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler): Promise { - const { module, apiBatches, totalCount, apiParams, concurrencyLimit } = env; - - /* eslint-disable no-async-promise-executor */ - return new Promise(async (resolve) => { - let batchNo = 0; - let isLastRequest = false; - const batch = fill(Array.from({ length: Number.parseInt(String(totalCount / 100), 10) }), 100); - - if (totalCount % 100) batch.push(100); - - const batches: Array = - apiBatches || - chunk( - map(batch, (skip: number, i: number) => skip * i), - concurrencyLimit, - ); - - /* eslint-disable no-promise-executor-return */ - if (isEmpty(batches)) return resolve(); - - for (const [batchIndex, batch] of entries(batches)) { - batchNo += 1; - const allPromise = []; - const start = Date.now(); - - for (const [index, element] of entries(batch)) { - let promise; - isLastRequest = isEqual(last(batch), element) && isEqual(last(batches), batch); - - if (promisifyHandler instanceof Function) { - promise = promisifyHandler({ - apiParams, - element, - isLastRequest, - index: Number(index), - batchIndex: Number(batchIndex), - }); - } else if (apiParams?.queryParam) { - apiParams.queryParam.skip = element; - promise = this.makeAPICall(apiParams, isLastRequest); - } - - allPromise.push(promise); - } - - /* eslint-disable no-await-in-loop */ - await Promise.allSettled(allPromise); - /* eslint-disable no-await-in-loop */ - await this.logMsgAndWaitIfRequired(module, start, batchNo); - - if (isLastRequest) resolve(); - } - }); - } - /** * @method logMsgAndWaitIfRequired * @param module string @@ -157,7 +115,7 @@ export default abstract class BaseClass { * @returns Promise */ makeAPICall( - { module: moduleName, reject, resolve, url = '', uid = '', additionalInfo, queryParam = {} }: ApiOptions, + { additionalInfo, module: moduleName, queryParam = {}, reject, resolve, uid = '', url = '' }: ApiOptions, isLastRequest = false, ): Promise { switch (moduleName) { @@ -165,21 +123,21 @@ export default abstract class BaseClass { return this.stack .asset(uid) .fetch(queryParam) - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => resolve({ additionalInfo, isLastRequest, response })) + .catch((error: Error) => reject({ additionalInfo, error, isLastRequest })); case 'assets': return this.stack .asset() .query(queryParam) .find() - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + .then((response: any) => resolve({ additionalInfo, isLastRequest, response })) + .catch((error: Error) => reject({ additionalInfo, error, isLastRequest })); case 'download-asset': return this.stack .asset() - .download({ url, responseType: 'stream' }) - .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) - .catch((error: any) => reject({ error, isLastRequest, additionalInfo })); + .download({ responseType: 'stream', url }) + .then((response: any) => resolve({ additionalInfo, isLastRequest, response })) + .catch((error: any) => reject({ additionalInfo, error, isLastRequest })); case 'export-taxonomy': return this.stack .taxonomy(uid) @@ -191,17 +149,59 @@ export default abstract class BaseClass { } } - protected applyQueryFilters(requestObject: any, moduleName: string): any { - if (this.exportConfig.query?.modules?.[moduleName]) { - const moduleQuery = this.exportConfig.query.modules[moduleName]; - // Merge the query parameters with existing requestObject - if (moduleQuery) { - if (!requestObject.query) { - requestObject.query = moduleQuery; + makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler): Promise { + const { apiBatches, apiParams, concurrencyLimit, module, totalCount } = env; + + /* eslint-disable no-async-promise-executor */ + return new Promise(async (resolve) => { + let batchNo = 0; + let isLastRequest = false; + const batch = fill(Array.from({ length: Number.parseInt(String(totalCount / 100), 10) }), 100); + + if (totalCount % 100) batch.push(100); + + const batches: Array = + apiBatches || + chunk( + map(batch, (skip: number, i: number) => skip * i), + concurrencyLimit, + ); + + /* eslint-disable no-promise-executor-return */ + if (isEmpty(batches)) return resolve(); + + for (const [batchIndex, batch] of entries(batches)) { + batchNo += 1; + const allPromise = []; + const start = Date.now(); + + for (const [index, element] of entries(batch)) { + let promise; + isLastRequest = isEqual(last(batch), element) && isEqual(last(batches), batch); + + if (promisifyHandler instanceof Function) { + promise = promisifyHandler({ + apiParams, + batchIndex: Number(batchIndex), + element, + index: Number(index), + isLastRequest, + }); + } else if (apiParams?.queryParam) { + apiParams.queryParam.skip = element; + promise = this.makeAPICall(apiParams, isLastRequest); + } + + allPromise.push(promise); } - Object.assign(requestObject.query, moduleQuery); + + /* eslint-disable no-await-in-loop */ + await Promise.allSettled(allPromise); + /* eslint-disable no-await-in-loop */ + await this.logMsgAndWaitIfRequired(module, start, batchNo); + + if (isLastRequest) resolve(); } - } - return requestObject; + }); } } diff --git a/packages/contentstack-export/src/export/modules/composable-studio.ts b/packages/contentstack-export/src/export/modules/composable-studio.ts index 8faff8c2b..143932efe 100644 --- a/packages/contentstack-export/src/export/modules/composable-studio.ts +++ b/packages/contentstack-export/src/export/modules/composable-studio.ts @@ -1,25 +1,25 @@ -import { resolve as pResolve } from 'node:path'; import { + HttpClient, + authenticationHandler, cliux, + handleAndLogError, isAuthenticated, log, messageHandler, - handleAndLogError, - HttpClient, - authenticationHandler, } from '@contentstack/cli-utilities'; +import { resolve as pResolve } from 'node:path'; +import { ComposableStudioConfig, ComposableStudioProject, ExportConfig, ModuleClassParams } from '../../types'; import { fsUtil, getOrgUid } from '../../utils'; -import { ModuleClassParams, ComposableStudioConfig, ExportConfig, ComposableStudioProject } from '../../types'; export default class ExportComposableStudio { - protected composableStudioConfig: ComposableStudioConfig; - protected composableStudioProject: ComposableStudioProject | null = null; protected apiClient: HttpClient; + protected composableStudioConfig: ComposableStudioConfig; public composableStudioPath: string; + protected composableStudioProject: ComposableStudioProject | null = null; public exportConfig: ExportConfig; - constructor({ exportConfig }: Omit) { + constructor({ exportConfig }: Omit) { this.exportConfig = exportConfig; this.composableStudioConfig = exportConfig.modules['composable-studio']; this.exportConfig.context.module = 'composable-studio'; @@ -29,34 +29,6 @@ export default class ExportComposableStudio { this.apiClient.baseUrl(`${this.composableStudioConfig.apiBaseUrl}/${this.composableStudioConfig.apiVersion}`); } - async start(): Promise { - log.debug('Starting Studio project export process...', this.exportConfig.context); - - if (!isAuthenticated()) { - cliux.print( - 'WARNING!!! To export Studio projects, you must be logged in. Please check csdx auth:login --help to log in', - { color: 'yellow' }, - ); - return Promise.resolve(); - } - - this.composableStudioPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.composableStudioConfig.dirName, - ); - log.debug(`Studio folder path: ${this.composableStudioPath}`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.composableStudioPath); - log.debug('Created Studio directory', this.exportConfig.context); - - this.exportConfig.org_uid = this.exportConfig.org_uid || (await getOrgUid(this.exportConfig)); - log.debug(`Organization UID: ${this.exportConfig.org_uid}`, this.exportConfig.context); - - await this.exportProjects(); - log.debug('Studio project export process completed', this.exportConfig.context); - } - /** * Export Studio projects connected to the current stack */ @@ -84,8 +56,8 @@ export default class ExportComposableStudio { // Set organization_uid header this.apiClient.headers({ - organization_uid: this.exportConfig.org_uid, Accept: 'application/json', + organization_uid: this.exportConfig.org_uid, }); const apiUrl = '/projects'; @@ -135,4 +107,32 @@ export default class ExportComposableStudio { }); } } + + async start(): Promise { + log.debug('Starting Studio project export process...', this.exportConfig.context); + + if (!isAuthenticated()) { + cliux.print( + 'WARNING!!! To export Studio projects, you must be logged in. Please check csdx auth:login --help to log in', + { color: 'yellow' }, + ); + return Promise.resolve(); + } + + this.composableStudioPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.composableStudioConfig.dirName, + ); + log.debug(`Studio folder path: ${this.composableStudioPath}`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.composableStudioPath); + log.debug('Created Studio directory', this.exportConfig.context); + + this.exportConfig.org_uid = this.exportConfig.org_uid || (await getOrgUid(this.exportConfig)); + log.debug(`Organization UID: ${this.exportConfig.org_uid}`, this.exportConfig.context); + + await this.exportProjects(); + log.debug('Studio project export process completed', this.exportConfig.context); + } } diff --git a/packages/contentstack-export/src/export/modules/content-types.ts b/packages/contentstack-export/src/export/modules/content-types.ts index ad571929d..04a1731e8 100644 --- a/packages/contentstack-export/src/export/modules/content-types.ts +++ b/packages/contentstack-export/src/export/modules/content-types.ts @@ -1,47 +1,47 @@ -import * as path from 'path'; import { ContentstackClient, handleAndLogError, - messageHandler, log, + messageHandler, sanitizePath, } from '@contentstack/cli-utilities'; +import * as path from 'path'; -import BaseClass from './base-class'; -import { fsUtil, executeTask } from '../../utils'; import { ExportConfig, ModuleClassParams } from '../../types'; +import { executeTask, fsUtil } from '../../utils'; +import BaseClass from './base-class'; export default class ContentTypesExport extends BaseClass { - private stackAPIClient: ReturnType; public exportConfig: ExportConfig; - private qs: { - include_count: boolean; - asc: string; - skip?: number; - limit?: number; - include_global_field_schema: boolean; - uid?: Record; - }; + private contentTypes: Record[]; private contentTypesConfig: { dirName?: string; + fetchConcurrency?: number; fileName?: string; + limit?: number; validKeys?: string[]; - fetchConcurrency?: number; writeConcurrency?: number; - limit?: number; }; private contentTypesDirPath: string; - private contentTypes: Record[]; + private qs: { + asc: string; + include_count: boolean; + include_global_field_schema: boolean; + limit?: number; + skip?: number; + uid?: Record; + }; + private stackAPIClient: ReturnType; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.stackAPIClient = stackAPIClient; this.contentTypesConfig = exportConfig.modules['content-types']; this.qs = { - include_count: true, asc: 'updated_at', - limit: this.contentTypesConfig.limit, + include_count: true, include_global_field_schema: true, + limit: this.contentTypesConfig.limit, }; // If content type id is provided then use it as part of query @@ -61,21 +61,6 @@ export default class ContentTypesExport extends BaseClass { this.exportConfig.context.module = 'content-types'; } - async start() { - try { - log.debug('Starting content types export process...', this.exportConfig.context); - await fsUtil.makeDirectory(this.contentTypesDirPath); - log.debug(`Created directory at: '${this.contentTypesDirPath}'.`, this.exportConfig.context); - - await this.getContentTypes(); - await this.writeContentTypes(this.contentTypes); - - log.success(messageHandler.parse('CONTENT_TYPE_EXPORT_COMPLETE'), this.exportConfig.context); - } catch (error) { - handleAndLogError(error, { ...this.exportConfig.context }); - } - } - async getContentTypes(skip = 0): Promise { if (skip) { this.qs.skip = skip; @@ -120,6 +105,21 @@ export default class ContentTypesExport extends BaseClass { return updatedContentTypes; } + async start() { + try { + log.debug('Starting content types export process...', this.exportConfig.context); + await fsUtil.makeDirectory(this.contentTypesDirPath); + log.debug(`Created directory at: '${this.contentTypesDirPath}'.`, this.exportConfig.context); + + await this.getContentTypes(); + await this.writeContentTypes(this.contentTypes); + + log.success(messageHandler.parse('CONTENT_TYPE_EXPORT_COMPLETE'), this.exportConfig.context); + } catch (error) { + handleAndLogError(error, { ...this.exportConfig.context }); + } + } + async writeContentTypes(contentTypes: Record[]) { log.debug(`Writing ${contentTypes?.length} content types to disk...`, this.exportConfig.context); diff --git a/packages/contentstack-export/src/export/modules/custom-roles.ts b/packages/contentstack-export/src/export/modules/custom-roles.ts index 1d9d10529..bbbe7d8bb 100644 --- a/packages/contentstack-export/src/export/modules/custom-roles.ts +++ b/packages/contentstack-export/src/export/modules/custom-roles.ts @@ -1,62 +1,39 @@ -import keys from 'lodash/keys'; +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import find from 'lodash/find'; import forEach from 'lodash/forEach'; +import keys from 'lodash/keys'; import values from 'lodash/values'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; -import { fsUtil } from '../../utils'; import { CustomRoleConfig, ModuleClassParams } from '../../types'; +import { fsUtil } from '../../utils'; +import BaseClass from './base-class'; export default class ExportCustomRoles extends BaseClass { + public customRolesLocalesFilepath: string; + public rolesFolderPath: string; private customRoles: Record; - private existingRoles: Record; private customRolesConfig: CustomRoleConfig; - private sourceLocalesMap: Record; + private existingRoles: Record; private localesMap: Record; - public rolesFolderPath: string; - public customRolesLocalesFilepath: string; + private sourceLocalesMap: Record; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.customRoles = {}; this.customRolesConfig = exportConfig.modules.customRoles; - this.existingRoles = { Admin: 1, Developer: 1, 'Content Manager': 1 }; + this.existingRoles = { Admin: 1, 'Content Manager': 1, Developer: 1 }; this.localesMap = {}; this.sourceLocalesMap = {}; this.exportConfig.context.module = 'custom-roles'; } - async start(): Promise { - log.debug('Starting export process for custom roles...', this.exportConfig.context); - - this.rolesFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.customRolesConfig.dirName, - ); - log.debug(`Custom roles folder path is: ${this.rolesFolderPath}`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.rolesFolderPath); - log.debug('Custom roles directory created.', this.exportConfig.context); - - this.customRolesLocalesFilepath = pResolve(this.rolesFolderPath, this.customRolesConfig.customRolesLocalesFileName); - log.debug(`Custom roles locales file path is: ${this.customRolesLocalesFilepath}`, this.exportConfig.context); - - await this.getCustomRoles(); - await this.getLocales(); - await this.getCustomRolesLocales(); - - log.debug(`Custom roles export completed. Total custom roles: ${Object.keys(this.customRoles).length}`, this.exportConfig.context); - } - async getCustomRoles(): Promise { log.debug('Fetching all roles from the stack...', this.exportConfig.context); const roles = await this.stack .role() - .fetchAll({ include_rules: true, include_permissions: true }) + .fetchAll({ include_permissions: true, include_rules: true }) .then((data: any) => { log.debug(`Fetched ${data.items?.length || 0} roles from the stack.`, this.exportConfig.context); return data; @@ -85,30 +62,6 @@ export default class ExportCustomRoles extends BaseClass { fsUtil.writeFile(customRolesFilePath, this.customRoles); } - async getLocales() { - log.debug('Fetching locales for custom roles mapping...', this.exportConfig.context); - - const locales = await this.stack - .locale() - .query({}) - .find() - .then((data: any) => { - log.debug(`Fetched ${data?.items?.length || 0} locales.`, this.exportConfig.context); - return data; - }) - .catch((err: any) => { - log.debug('An error occurred while fetching locales.', this.exportConfig.context); - return handleAndLogError(err, { ...this.exportConfig.context }); - }); - - for (const locale of locales.items) { - log.debug(`Mapping locale: ${locale?.name} (${locale?.uid})`, this.exportConfig.context); - this.sourceLocalesMap[locale.uid] = locale; - } - - log.debug(`Mapped ${Object.keys(this.sourceLocalesMap).length} source locales.`, this.exportConfig.context); - } - async getCustomRolesLocales() { log.debug('Processing custom roles locales mapping...', this.exportConfig.context); @@ -144,4 +97,51 @@ export default class ExportCustomRoles extends BaseClass { log.debug('No custom role locales found to process.', this.exportConfig.context); } } + + async getLocales() { + log.debug('Fetching locales for custom roles mapping...', this.exportConfig.context); + + const locales = await this.stack + .locale() + .query({}) + .find() + .then((data: any) => { + log.debug(`Fetched ${data?.items?.length || 0} locales.`, this.exportConfig.context); + return data; + }) + .catch((err: any) => { + log.debug('An error occurred while fetching locales.', this.exportConfig.context); + return handleAndLogError(err, { ...this.exportConfig.context }); + }); + + for (const locale of locales.items) { + log.debug(`Mapping locale: ${locale?.name} (${locale?.uid})`, this.exportConfig.context); + this.sourceLocalesMap[locale.uid] = locale; + } + + log.debug(`Mapped ${Object.keys(this.sourceLocalesMap).length} source locales.`, this.exportConfig.context); + } + + async start(): Promise { + log.debug('Starting export process for custom roles...', this.exportConfig.context); + + this.rolesFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.customRolesConfig.dirName, + ); + log.debug(`Custom roles folder path is: ${this.rolesFolderPath}`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.rolesFolderPath); + log.debug('Custom roles directory created.', this.exportConfig.context); + + this.customRolesLocalesFilepath = pResolve(this.rolesFolderPath, this.customRolesConfig.customRolesLocalesFileName); + log.debug(`Custom roles locales file path is: ${this.customRolesLocalesFilepath}`, this.exportConfig.context); + + await this.getCustomRoles(); + await this.getLocales(); + await this.getCustomRolesLocales(); + + log.debug(`Custom roles export completed. Total custom roles: ${Object.keys(this.customRoles).length}`, this.exportConfig.context); + } } diff --git a/packages/contentstack-export/src/export/modules/entries.ts b/packages/contentstack-export/src/export/modules/entries.ts index 743b471ea..5afbb2127 100644 --- a/packages/contentstack-export/src/export/modules/entries.ts +++ b/packages/contentstack-export/src/export/modules/entries.ts @@ -1,33 +1,32 @@ -import * as path from 'path'; -import { ContentstackClient, FsUtility, handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; +import { ContentstackClient, FsUtility, handleAndLogError, log, messageHandler , sanitizePath } from '@contentstack/cli-utilities'; import { Export, ExportProjects } from '@contentstack/cli-variants'; -import { sanitizePath } from '@contentstack/cli-utilities'; +import * as path from 'path'; +import { ExportConfig, ModuleClassParams } from '../../types'; import { fsUtil } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; -import { ExportConfig, ModuleClassParams } from '../../types'; export default class EntriesExport extends BaseClass { - private stackAPIClient: ReturnType; public exportConfig: ExportConfig; + public exportVariantEntry = false; private entriesConfig: { + batchLimit?: number; + chunkFileSize?: number; dirName?: string; + exportVersions: boolean; + fetchConcurrency?: number; fileName?: string; invalidKeys?: string[]; - fetchConcurrency?: number; - writeConcurrency?: number; limit?: number; - chunkFileSize?: number; - batchLimit?: number; - exportVersions: boolean; + writeConcurrency?: number; }; - private variantEntries!: any; private entriesDirPath: string; - private localesFilePath: string; - private schemaFilePath: string; private entriesFileHelper: FsUtility; + private localesFilePath: string; private projectInstance: ExportProjects; - public exportVariantEntry: boolean = false; + private schemaFilePath: string; + private stackAPIClient: ReturnType; + private variantEntries!: any; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -55,66 +54,6 @@ export default class EntriesExport extends BaseClass { this.exportConfig.context.module = 'entries'; } - async start() { - try { - log.debug('Starting entries export process...', this.exportConfig.context); - const locales = fsUtil.readFile(this.localesFilePath) as Array>; - if (!Array.isArray(locales) || locales?.length === 0) { - log.debug(`No locales found in ${this.localesFilePath}`, this.exportConfig.context); - } else { - log.debug(`Loaded ${locales?.length} locales from ${this.localesFilePath}`, this.exportConfig.context); - } - - const contentTypes = fsUtil.readFile(this.schemaFilePath) as Array>; - if (contentTypes?.length === 0) { - log.info(messageHandler.parse('CONTENT_TYPE_NO_TYPES'), this.exportConfig.context); - return; - } - log.debug(`Loaded ${contentTypes?.length} content types from ${this.schemaFilePath}`, this.exportConfig.context); - - // NOTE Check if variant is enabled in specific stack - if (this.exportConfig.personalizationEnabled) { - log.debug('Personalization is enabled, checking for variant entries...', this.exportConfig.context); - let project_id; - try { - const project = await this.projectInstance.projects({ connectedStackApiKey: this.exportConfig.apiKey }); - - if (project && project[0]?.uid) { - project_id = project[0].uid; - this.exportVariantEntry = true; - log.debug(`Found project with ID: ${project_id}, enabling variant entry export`, this.exportConfig.context); - } - - this.variantEntries = new Export.VariantEntries(Object.assign(this.exportConfig, { project_id })); - } catch (error) { - handleAndLogError(error, { ...this.exportConfig.context }); - } - } - - const entryRequestOptions = this.createRequestObjects(locales, contentTypes); - log.debug( - `Created ${entryRequestOptions.length} entry request objects for processing`, - this.exportConfig.context, - ); - - for (let entryRequestOption of entryRequestOptions) { - log.debug( - `Processing entries for content type: ${entryRequestOption.contentType}, locale: ${entryRequestOption.locale}`, - this.exportConfig.context, - ); - await this.getEntries(entryRequestOption); - this.entriesFileHelper?.completeFile(true); - log.success( - messageHandler.parse('ENTRIES_EXPORT_COMPLETE', entryRequestOption.contentType, entryRequestOption.locale), - this.exportConfig.context, - ); - } - log.success(messageHandler.parse('ENTRIES_EXPORT_SUCCESS'), this.exportConfig.context); - } catch (error) { - handleAndLogError(error, { ...this.exportConfig.context }); - } - } - createRequestObjects( locales: Array>, contentTypes: Array>, @@ -131,10 +70,10 @@ export default class EntriesExport extends BaseClass { log.debug(`Found ${contentTypes.length} content types for export`, this.exportConfig.context); } - let requestObjects: Array> = []; + const requestObjects: Array> = []; contentTypes.forEach((contentType) => { if (Object.keys(locales).length !== 0) { - for (let locale in locales) { + for (const locale in locales) { requestObjects.push({ contentType: contentType.uid, locale: locales[locale].code, @@ -151,17 +90,96 @@ export default class EntriesExport extends BaseClass { return requestObjects; } + async entryVersionHandler({ + apiParams, + element: entry, + }: { + apiParams: ApiOptions; + element: Record; + isLastRequest: boolean; + }) { + log.debug(`Processing versioned entry: ${entry.uid}`, this.exportConfig.context); + + return new Promise(async (resolve, reject) => { + return await this.getEntryByVersion(apiParams.queryParam, entry) + .then((response) => { + log.debug(`Successfully fetched versions for entry UID: ${entry.uid}`, this.exportConfig.context); + apiParams.resolve({ + apiData: entry, + response, + }); + resolve(true); + }) + .catch((error) => { + log.debug(`Failed to fetch versions for entry UID: ${entry.uid}`, this.exportConfig.context); + apiParams.reject({ + apiData: entry, + error, + }); + reject(true); + }); + }); + } + + async fetchEntriesVersions( + entries: any, + options: { contentType: string; locale: string; versionedEntryPath: string }, + ): Promise { + log.debug(`Fetching versions for ${entries.length} entries...`, this.exportConfig.context); + + const onSuccess = ({ apiData: entry, response }: any) => { + const versionFilePath = path.join(sanitizePath(options.versionedEntryPath), sanitizePath(`${entry.uid}.json`)); + log.debug(`Writing versioned entry to: ${versionFilePath}`, this.exportConfig.context); + fsUtil.writeFile(versionFilePath, response); + log.success( + messageHandler.parse('ENTRIES_VERSIONED_EXPORT_SUCCESS', options.contentType, entry.uid, options.locale), + this.exportConfig.context, + ); + }; + const onReject = ({ apiData: { uid } = undefined, error }: any) => { + log.debug(`Failed to fetch versioned entry for uid: ${uid}`, this.exportConfig.context); + handleAndLogError( + error, + { + ...this.exportConfig.context, + uid, + }, + messageHandler.parse('ENTRIES_EXPORT_VERSIONS_FAILED', uid), + ); + }; + + log.debug( + `Starting concurrent calls for versioned entries with batch limit: ${this.entriesConfig.batchLimit}`, + this.exportConfig.context, + ); + return await this.makeConcurrentCall( + { + apiBatches: [entries], + apiParams: { + module: 'versioned-entries', + queryParam: options, + reject: onReject, + resolve: onSuccess, + }, + concurrencyLimit: this.entriesConfig.batchLimit, + module: 'versioned-entries', + totalCount: entries.length, + }, + this.entryVersionHandler.bind(this), + ); + } + async getEntries(options: Record): Promise { options.skip = options.skip || 0; - let requestObject = { - locale: options.locale, - skip: options.skip, - limit: this.entriesConfig.limit, + const requestObject = { include_count: true, include_publish_details: true, + limit: this.entriesConfig.limit, + locale: options.locale, query: { locale: options.locale, }, + skip: options.skip, }; this.applyQueryFilters(requestObject, 'entries'); @@ -198,11 +216,11 @@ export default class EntriesExport extends BaseClass { log.debug(`Creating directory for entries at: ${entryBasePath}`, this.exportConfig.context); await fsUtil.makeDirectory(entryBasePath); this.entriesFileHelper = new FsUtility({ - moduleName: 'entries', - indexFileName: 'index.json', basePath: entryBasePath, chunkFileSize: this.entriesConfig.chunkFileSize, + indexFileName: 'index.json', keepMetadata: false, + moduleName: 'entries', omitKeys: this.entriesConfig.invalidKeys, }); log.debug('Initialized FsUtility for writing entries', this.exportConfig.context); @@ -213,7 +231,7 @@ export default class EntriesExport extends BaseClass { if (this.entriesConfig.exportVersions) { log.debug('Exporting entry versions is enabled.', this.exportConfig.context); - let versionedEntryPath = path.join( + const versionedEntryPath = path.join( sanitizePath(this.entriesDirPath), sanitizePath(options.contentType), sanitizePath(options.locale), @@ -222,8 +240,8 @@ export default class EntriesExport extends BaseClass { log.debug(`Creating versioned entries directory at: ${versionedEntryPath}.`, this.exportConfig.context); fsUtil.makeDirectory(versionedEntryPath); await this.fetchEntriesVersions(entriesSearchResponse.items, { - locale: options.locale, contentType: options.contentType, + locale: options.locale, versionedEntryPath, }); } @@ -232,9 +250,9 @@ export default class EntriesExport extends BaseClass { if (this.exportVariantEntry) { log.debug('Exporting variant entries for base entries', this.exportConfig.context); await this.variantEntries.exportVariantEntry({ - locale: options.locale, contentTypeUid: options.contentType, entries: entriesSearchResponse.items, + locale: options.locale, }); } @@ -251,95 +269,16 @@ export default class EntriesExport extends BaseClass { } } - async fetchEntriesVersions( - entries: any, - options: { locale: string; contentType: string; versionedEntryPath: string }, - ): Promise { - log.debug(`Fetching versions for ${entries.length} entries...`, this.exportConfig.context); - - const onSuccess = ({ response, apiData: entry }: any) => { - const versionFilePath = path.join(sanitizePath(options.versionedEntryPath), sanitizePath(`${entry.uid}.json`)); - log.debug(`Writing versioned entry to: ${versionFilePath}`, this.exportConfig.context); - fsUtil.writeFile(versionFilePath, response); - log.success( - messageHandler.parse('ENTRIES_VERSIONED_EXPORT_SUCCESS', options.contentType, entry.uid, options.locale), - this.exportConfig.context, - ); - }; - const onReject = ({ error, apiData: { uid } = undefined }: any) => { - log.debug(`Failed to fetch versioned entry for uid: ${uid}`, this.exportConfig.context); - handleAndLogError( - error, - { - ...this.exportConfig.context, - uid, - }, - messageHandler.parse('ENTRIES_EXPORT_VERSIONS_FAILED', uid), - ); - }; - - log.debug( - `Starting concurrent calls for versioned entries with batch limit: ${this.entriesConfig.batchLimit}`, - this.exportConfig.context, - ); - return await this.makeConcurrentCall( - { - apiBatches: [entries], - module: 'versioned-entries', - totalCount: entries.length, - concurrencyLimit: this.entriesConfig.batchLimit, - apiParams: { - module: 'versioned-entries', - queryParam: options, - resolve: onSuccess, - reject: onReject, - }, - }, - this.entryVersionHandler.bind(this), - ); - } - - async entryVersionHandler({ - apiParams, - element: entry, - }: { - apiParams: ApiOptions; - element: Record; - isLastRequest: boolean; - }) { - log.debug(`Processing versioned entry: ${entry.uid}`, this.exportConfig.context); - - return new Promise(async (resolve, reject) => { - return await this.getEntryByVersion(apiParams.queryParam, entry) - .then((response) => { - log.debug(`Successfully fetched versions for entry UID: ${entry.uid}`, this.exportConfig.context); - apiParams.resolve({ - response, - apiData: entry, - }); - resolve(true); - }) - .catch((error) => { - log.debug(`Failed to fetch versions for entry UID: ${entry.uid}`, this.exportConfig.context); - apiParams.reject({ - error, - apiData: entry, - }); - reject(true); - }); - }); - } - async getEntryByVersion( options: any, entry: Record, entries: Array> = [], ): Promise { const queryRequestObject = { - locale: options.locale, except: { BASE: this.entriesConfig.invalidKeys, }, + locale: options.locale, version: entry._version, }; @@ -365,4 +304,64 @@ export default class EntriesExport extends BaseClass { ); return entries; } + + async start() { + try { + log.debug('Starting entries export process...', this.exportConfig.context); + const locales = fsUtil.readFile(this.localesFilePath) as Array>; + if (!Array.isArray(locales) || locales?.length === 0) { + log.debug(`No locales found in ${this.localesFilePath}`, this.exportConfig.context); + } else { + log.debug(`Loaded ${locales?.length} locales from ${this.localesFilePath}`, this.exportConfig.context); + } + + const contentTypes = fsUtil.readFile(this.schemaFilePath) as Array>; + if (contentTypes?.length === 0) { + log.info(messageHandler.parse('CONTENT_TYPE_NO_TYPES'), this.exportConfig.context); + return; + } + log.debug(`Loaded ${contentTypes?.length} content types from ${this.schemaFilePath}`, this.exportConfig.context); + + // NOTE Check if variant is enabled in specific stack + if (this.exportConfig.personalizationEnabled) { + log.debug('Personalization is enabled, checking for variant entries...', this.exportConfig.context); + let project_id; + try { + const project = await this.projectInstance.projects({ connectedStackApiKey: this.exportConfig.apiKey }); + + if (project && project[0]?.uid) { + project_id = project[0].uid; + this.exportVariantEntry = true; + log.debug(`Found project with ID: ${project_id}, enabling variant entry export`, this.exportConfig.context); + } + + this.variantEntries = new Export.VariantEntries(Object.assign(this.exportConfig, { project_id })); + } catch (error) { + handleAndLogError(error, { ...this.exportConfig.context }); + } + } + + const entryRequestOptions = this.createRequestObjects(locales, contentTypes); + log.debug( + `Created ${entryRequestOptions.length} entry request objects for processing`, + this.exportConfig.context, + ); + + for (const entryRequestOption of entryRequestOptions) { + log.debug( + `Processing entries for content type: ${entryRequestOption.contentType}, locale: ${entryRequestOption.locale}`, + this.exportConfig.context, + ); + await this.getEntries(entryRequestOption); + this.entriesFileHelper?.completeFile(true); + log.success( + messageHandler.parse('ENTRIES_EXPORT_COMPLETE', entryRequestOption.contentType, entryRequestOption.locale), + this.exportConfig.context, + ); + } + log.success(messageHandler.parse('ENTRIES_EXPORT_SUCCESS'), this.exportConfig.context); + } catch (error) { + handleAndLogError(error, { ...this.exportConfig.context }); + } + } } diff --git a/packages/contentstack-export/src/export/modules/environments.ts b/packages/contentstack-export/src/export/modules/environments.ts index 85e9f8af3..b0538c839 100644 --- a/packages/contentstack-export/src/export/modules/environments.ts +++ b/packages/contentstack-export/src/export/modules/environments.ts @@ -1,16 +1,16 @@ -import { resolve as pResolve } from 'node:path'; -import omit from 'lodash/omit'; +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import isEmpty from 'lodash/isEmpty'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; +import omit from 'lodash/omit'; +import { resolve as pResolve } from 'node:path'; -import BaseClass from './base-class'; -import { fsUtil } from '../../utils'; import { EnvironmentConfig, ModuleClassParams } from '../../types'; +import { fsUtil } from '../../utils'; +import BaseClass from './base-class'; export default class ExportEnvironments extends BaseClass { - private environments: Record; - private environmentConfig: EnvironmentConfig; public environmentsFolderPath: string; + private environmentConfig: EnvironmentConfig; + private environments: Record; private qs: { include_count: boolean; skip?: number; @@ -24,34 +24,6 @@ export default class ExportEnvironments extends BaseClass { this.exportConfig.context.module = 'environments'; } - async start(): Promise { - log.debug('Starting environment export process...', this.exportConfig.context); - this.environmentsFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.environmentConfig.dirName, - ); - log.debug(`Environments folder path is: ${this.environmentsFolderPath}`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.environmentsFolderPath); - log.debug('Environments directory created.', this.exportConfig.context); - - await this.getEnvironments(); - log.debug(`Retrieved ${Object.keys(this.environments).length} environments.`, this.exportConfig.context); - - if (this.environments === undefined || isEmpty(this.environments)) { - log.info(messageHandler.parse('ENVIRONMENT_NOT_FOUND'), this.exportConfig.context); - } else { - const environmentsFilePath = pResolve(this.environmentsFolderPath, this.environmentConfig.fileName); - log.debug(`Writing environments to: ${environmentsFilePath}.`, this.exportConfig.context); - fsUtil.writeFile(environmentsFilePath, this.environments); - log.success( - messageHandler.parse('ENVIRONMENT_EXPORT_COMPLETE', Object.keys(this.environments).length), - this.exportConfig.context, - ); - } - } - async getEnvironments(skip = 0): Promise { if (skip) { this.qs.skip = skip; @@ -67,7 +39,7 @@ export default class ExportEnvironments extends BaseClass { .query(this.qs) .find() .then(async (data: any) => { - const { items, count } = data; + const { count, items } = data; log.debug(`Fetched ${items?.length || 0} environments out of ${count} total.`, this.exportConfig.context); if (items?.length) { @@ -104,4 +76,32 @@ export default class ExportEnvironments extends BaseClass { log.debug(`Sanitization complete. Total environments processed: ${Object.keys(this.environments).length}`, this.exportConfig.context); } + + async start(): Promise { + log.debug('Starting environment export process...', this.exportConfig.context); + this.environmentsFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.environmentConfig.dirName, + ); + log.debug(`Environments folder path is: ${this.environmentsFolderPath}`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.environmentsFolderPath); + log.debug('Environments directory created.', this.exportConfig.context); + + await this.getEnvironments(); + log.debug(`Retrieved ${Object.keys(this.environments).length} environments.`, this.exportConfig.context); + + if (this.environments === undefined || isEmpty(this.environments)) { + log.info(messageHandler.parse('ENVIRONMENT_NOT_FOUND'), this.exportConfig.context); + } else { + const environmentsFilePath = pResolve(this.environmentsFolderPath, this.environmentConfig.fileName); + log.debug(`Writing environments to: ${environmentsFilePath}.`, this.exportConfig.context); + fsUtil.writeFile(environmentsFilePath, this.environments); + log.success( + messageHandler.parse('ENVIRONMENT_EXPORT_COMPLETE', Object.keys(this.environments).length), + this.exportConfig.context, + ); + } + } } diff --git a/packages/contentstack-export/src/export/modules/extensions.ts b/packages/contentstack-export/src/export/modules/extensions.ts index 5da610ec2..2abfa827b 100644 --- a/packages/contentstack-export/src/export/modules/extensions.ts +++ b/packages/contentstack-export/src/export/modules/extensions.ts @@ -1,16 +1,16 @@ -import omit from 'lodash/omit'; +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import isEmpty from 'lodash/isEmpty'; +import omit from 'lodash/omit'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; -import { fsUtil } from '../../utils'; import { ExtensionsConfig, ModuleClassParams } from '../../types'; +import { fsUtil } from '../../utils'; +import BaseClass from './base-class'; export default class ExportExtensions extends BaseClass { - private extensionsFolderPath: string; - private extensions: Record; public extensionConfig: ExtensionsConfig; + private extensions: Record; + private extensionsFolderPath: string; private qs: { include_count: boolean; skip?: number; @@ -25,35 +25,6 @@ export default class ExportExtensions extends BaseClass { this.exportConfig.context.module = 'extensions'; } - async start(): Promise { - log.debug('Starting extensions export process...', this.exportConfig.context); - - this.extensionsFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.extensionConfig.dirName, - ); - log.debug(`Extensions folder path is: ${this.extensionsFolderPath}`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.extensionsFolderPath); - log.debug('Extensions directory created.', this.exportConfig.context); - - await this.getExtensions(); - log.debug(`Retrieved ${Object.keys(this.extensions).length} extensions.`, this.exportConfig.context); - - if (this.extensions === undefined || isEmpty(this.extensions)) { - log.info(messageHandler.parse('EXTENSION_NOT_FOUND'), this.exportConfig.context); - } else { - const extensionsFilePath = pResolve(this.extensionsFolderPath, this.extensionConfig.fileName); - log.debug(`Writing extensions to: ${extensionsFilePath}.`, this.exportConfig.context); - fsUtil.writeFile(extensionsFilePath, this.extensions); - log.success( - messageHandler.parse('EXTENSION_EXPORT_COMPLETE', Object.keys(this.extensions).length ), - this.exportConfig.context, - ); - } - } - async getExtensions(skip = 0): Promise { if (skip) { this.qs.skip = skip; @@ -69,7 +40,7 @@ export default class ExportExtensions extends BaseClass { .query(this.qs) .find() .then(async (data: any) => { - const { items, count } = data; + const { count, items } = data; log.debug(`Fetched ${items?.length || 0} extensions out of ${count}.`, this.exportConfig.context); if (items?.length) { @@ -106,4 +77,33 @@ export default class ExportExtensions extends BaseClass { log.debug(`Sanitization complete. Total extensions processed: ${Object.keys(this.extensions).length}.`, this.exportConfig.context); } + + async start(): Promise { + log.debug('Starting extensions export process...', this.exportConfig.context); + + this.extensionsFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.extensionConfig.dirName, + ); + log.debug(`Extensions folder path is: ${this.extensionsFolderPath}`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.extensionsFolderPath); + log.debug('Extensions directory created.', this.exportConfig.context); + + await this.getExtensions(); + log.debug(`Retrieved ${Object.keys(this.extensions).length} extensions.`, this.exportConfig.context); + + if (this.extensions === undefined || isEmpty(this.extensions)) { + log.info(messageHandler.parse('EXTENSION_NOT_FOUND'), this.exportConfig.context); + } else { + const extensionsFilePath = pResolve(this.extensionsFolderPath, this.extensionConfig.fileName); + log.debug(`Writing extensions to: ${extensionsFilePath}.`, this.exportConfig.context); + fsUtil.writeFile(extensionsFilePath, this.extensions); + log.success( + messageHandler.parse('EXTENSION_EXPORT_COMPLETE', Object.keys(this.extensions).length ), + this.exportConfig.context, + ); + } + } } diff --git a/packages/contentstack-export/src/export/modules/global-fields.ts b/packages/contentstack-export/src/export/modules/global-fields.ts index 421665cfc..93c4e5a45 100644 --- a/packages/contentstack-export/src/export/modules/global-fields.ts +++ b/packages/contentstack-export/src/export/modules/global-fields.ts @@ -1,47 +1,47 @@ -import * as path from 'path'; import { ContentstackClient, handleAndLogError, - messageHandler, log, + messageHandler, sanitizePath, } from '@contentstack/cli-utilities'; +import * as path from 'path'; -import { fsUtil } from '../../utils'; import { ExportConfig, ModuleClassParams } from '../../types'; +import { fsUtil } from '../../utils'; import BaseClass from './base-class'; export default class GlobalFieldsExport extends BaseClass { - private stackAPIClient: ReturnType; public exportConfig: ExportConfig; - private qs: { - include_count: boolean; - asc: string; - skip?: number; - limit?: number; - include_global_field_schema?: boolean; - }; + private globalFields: Record[]; private globalFieldsConfig: { dirName?: string; + fetchConcurrency?: number; fileName?: string; + limit?: number; validKeys?: string[]; - fetchConcurrency?: number; writeConcurrency?: number; - limit?: number; }; private globalFieldsDirPath: string; - private globalFields: Record[]; + private qs: { + asc: string; + include_count: boolean; + include_global_field_schema?: boolean; + limit?: number; + skip?: number; + }; + private stackAPIClient: ReturnType; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.stackAPIClient = stackAPIClient; this.globalFieldsConfig = exportConfig.modules['global-fields']; this.qs = { - skip: 0, asc: 'updated_at', include_count: true, - limit: this.globalFieldsConfig.limit, include_global_field_schema: true, + limit: this.globalFieldsConfig.limit, + skip: 0, }; this.globalFieldsDirPath = path.resolve( sanitizePath(exportConfig.data), @@ -53,38 +53,14 @@ export default class GlobalFieldsExport extends BaseClass { this.exportConfig.context.module = 'global-fields'; } - async start() { - try { - log.debug('Starting export process for global fields...', this.exportConfig.context); - log.debug(`Global fields directory path: '${this.globalFieldsDirPath}'`, this.exportConfig.context); - await fsUtil.makeDirectory(this.globalFieldsDirPath); - log.debug('Created global fields directory.', this.exportConfig.context); - - await this.getGlobalFields(); - log.debug(`Retrieved ${this.globalFields.length} global fields.`, this.exportConfig.context); - - const globalFieldsFilePath = path.join(this.globalFieldsDirPath, this.globalFieldsConfig.fileName); - log.debug(`Writing global fields to: '${globalFieldsFilePath}'`, this.exportConfig.context); - fsUtil.writeFile(globalFieldsFilePath, this.globalFields); - - log.success( - messageHandler.parse('GLOBAL_FIELDS_EXPORT_COMPLETE', this.globalFields.length), - this.exportConfig.context, - ); - } catch (error) { - log.debug('An error occurred during global fields export.', this.exportConfig.context); - handleAndLogError(error, { ...this.exportConfig.context }); - } - } - - async getGlobalFields(skip: number = 0): Promise { + async getGlobalFields(skip = 0): Promise { if (skip) { this.qs.skip = skip; log.debug(`Fetching global fields with skip: ${skip}.`, this.exportConfig.context); } log.debug(`Query parameters: ${JSON.stringify(this.qs)}.`, this.exportConfig.context); - let globalFieldsFetchResponse = await this.stackAPIClient.globalField({ api_version: '3.2' }).query(this.qs).find(); + const globalFieldsFetchResponse = await this.stackAPIClient.globalField({ api_version: '3.2' }).query(this.qs).find(); log.debug(`Fetched ${globalFieldsFetchResponse.items?.length || 0} global fields out of ${globalFieldsFetchResponse.count}.`, this.exportConfig.context); @@ -109,7 +85,7 @@ export default class GlobalFieldsExport extends BaseClass { globalFields.forEach((globalField: Record) => { log.debug(`Processing global field: '${globalField.uid || 'unknown'}'...`, this.exportConfig.context); - for (let key in globalField) { + for (const key in globalField) { if (this.globalFieldsConfig.validKeys.indexOf(key) === -1) { delete globalField[key]; } @@ -119,4 +95,28 @@ export default class GlobalFieldsExport extends BaseClass { log.debug(`Sanitization complete. Total global fields processed: ${this.globalFields.length}.`, this.exportConfig.context); } + + async start() { + try { + log.debug('Starting export process for global fields...', this.exportConfig.context); + log.debug(`Global fields directory path: '${this.globalFieldsDirPath}'`, this.exportConfig.context); + await fsUtil.makeDirectory(this.globalFieldsDirPath); + log.debug('Created global fields directory.', this.exportConfig.context); + + await this.getGlobalFields(); + log.debug(`Retrieved ${this.globalFields.length} global fields.`, this.exportConfig.context); + + const globalFieldsFilePath = path.join(this.globalFieldsDirPath, this.globalFieldsConfig.fileName); + log.debug(`Writing global fields to: '${globalFieldsFilePath}'`, this.exportConfig.context); + fsUtil.writeFile(globalFieldsFilePath, this.globalFields); + + log.success( + messageHandler.parse('GLOBAL_FIELDS_EXPORT_COMPLETE', this.globalFields.length), + this.exportConfig.context, + ); + } catch (error) { + log.debug('An error occurred during global fields export.', this.exportConfig.context); + handleAndLogError(error, { ...this.exportConfig.context }); + } + } } diff --git a/packages/contentstack-export/src/export/modules/index.ts b/packages/contentstack-export/src/export/modules/index.ts index f13bc4fa3..e9025b1cd 100644 --- a/packages/contentstack-export/src/export/modules/index.ts +++ b/packages/contentstack-export/src/export/modules/index.ts @@ -1,4 +1,5 @@ import { handleAndLogError } from '@contentstack/cli-utilities'; + import { ModuleClassParams } from '../../types'; export default async function startModuleExport(modulePayload: ModuleClassParams) { diff --git a/packages/contentstack-export/src/export/modules/labels.ts b/packages/contentstack-export/src/export/modules/labels.ts index 414f13077..a9c773973 100644 --- a/packages/contentstack-export/src/export/modules/labels.ts +++ b/packages/contentstack-export/src/export/modules/labels.ts @@ -1,16 +1,16 @@ -import omit from 'lodash/omit'; +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import isEmpty from 'lodash/isEmpty'; +import omit from 'lodash/omit'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; -import { fsUtil } from '../../utils'; import { LabelConfig, ModuleClassParams } from '../../types'; +import { fsUtil } from '../../utils'; +import BaseClass from './base-class'; export default class ExportLabels extends BaseClass { - private labels: Record>; - private labelConfig: LabelConfig; public labelsFolderPath: string; + private labelConfig: LabelConfig; + private labels: Record>; private qs: { include_count: boolean; skip?: number; @@ -24,35 +24,6 @@ export default class ExportLabels extends BaseClass { this.exportConfig.context.module = 'labels'; } - async start(): Promise { - log.debug('Starting export process for labels...', this.exportConfig.context); - - this.labelsFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.labelConfig.dirName, - ); - log.debug(`Labels folder path: '${this.labelsFolderPath}'`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.labelsFolderPath); - log.debug('Created labels directory.', this.exportConfig.context); - - await this.getLabels(); - log.debug(`Retrieved ${Object.keys(this.labels).length} labels.`, this.exportConfig.context); - - if (this.labels === undefined || isEmpty(this.labels)) { - log.info(messageHandler.parse('LABELS_NOT_FOUND'), this.exportConfig.context); - } else { - const labelsFilePath = pResolve(this.labelsFolderPath, this.labelConfig.fileName); - log.debug(`Writing labels to: '${labelsFilePath}'.`, this.exportConfig.context); - fsUtil.writeFile(labelsFilePath, this.labels); - log.success( - messageHandler.parse('LABELS_EXPORT_COMPLETE', Object.keys(this.labels).length), - this.exportConfig.context, - ); - } - } - async getLabels(skip = 0): Promise { if (skip) { this.qs.skip = skip; @@ -68,7 +39,7 @@ export default class ExportLabels extends BaseClass { .query(this.qs) .find() .then(async (data: any) => { - const { items, count } = data; + const { count, items } = data; log.debug(`Fetched ${items?.length || 0} labels out of ${count}.`, this.exportConfig.context); if (items?.length) { @@ -105,4 +76,33 @@ export default class ExportLabels extends BaseClass { log.debug(`Sanitization complete. Total labels processed: ${Object.keys(this.labels).length}.`, this.exportConfig.context); } + + async start(): Promise { + log.debug('Starting export process for labels...', this.exportConfig.context); + + this.labelsFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.labelConfig.dirName, + ); + log.debug(`Labels folder path: '${this.labelsFolderPath}'`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.labelsFolderPath); + log.debug('Created labels directory.', this.exportConfig.context); + + await this.getLabels(); + log.debug(`Retrieved ${Object.keys(this.labels).length} labels.`, this.exportConfig.context); + + if (this.labels === undefined || isEmpty(this.labels)) { + log.info(messageHandler.parse('LABELS_NOT_FOUND'), this.exportConfig.context); + } else { + const labelsFilePath = pResolve(this.labelsFolderPath, this.labelConfig.fileName); + log.debug(`Writing labels to: '${labelsFilePath}'.`, this.exportConfig.context); + fsUtil.writeFile(labelsFilePath, this.labels); + log.success( + messageHandler.parse('LABELS_EXPORT_COMPLETE', Object.keys(this.labels).length), + this.exportConfig.context, + ); + } + } } diff --git a/packages/contentstack-export/src/export/modules/locales.ts b/packages/contentstack-export/src/export/modules/locales.ts index 97f8d16fe..9f522f6d9 100644 --- a/packages/contentstack-export/src/export/modules/locales.ts +++ b/packages/contentstack-export/src/export/modules/locales.ts @@ -1,39 +1,39 @@ -import * as path from 'path'; import { ContentstackClient, handleAndLogError, - messageHandler, log, + messageHandler, sanitizePath, } from '@contentstack/cli-utilities'; +import * as path from 'path'; +import { ExportConfig, ModuleClassParams } from '../../types'; import { fsUtil } from '../../utils'; import BaseClass from './base-class'; -import { ExportConfig, ModuleClassParams } from '../../types'; export default class LocaleExport extends BaseClass { - private stackAPIClient: ReturnType; public exportConfig: ExportConfig; - private masterLocaleConfig: { dirName: string; fileName: string; requiredKeys: string[] }; - private qs: { - include_count: boolean; - asc: string; - only: { - BASE: string[]; - }; - skip?: number; - }; private localeConfig: { dirName?: string; + fetchConcurrency?: number; fileName?: string; + limit?: number; requiredKeys?: string[]; - fetchConcurrency?: number; writeConcurrency?: number; - limit?: number; }; + private locales: Record>; private localesPath: string; private masterLocale: Record>; - private locales: Record>; + private masterLocaleConfig: { dirName: string; fileName: string; requiredKeys: string[] }; + private qs: { + asc: string; + include_count: boolean; + only: { + BASE: string[]; + }; + skip?: number; + }; + private stackAPIClient: ReturnType; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -41,8 +41,8 @@ export default class LocaleExport extends BaseClass { this.localeConfig = exportConfig.modules.locales; this.masterLocaleConfig = exportConfig.modules.masterLocale; this.qs = { - include_count: true, asc: 'updated_at', + include_count: true, only: { BASE: this.localeConfig.requiredKeys, }, @@ -57,48 +57,14 @@ export default class LocaleExport extends BaseClass { this.exportConfig.context.module = 'locales'; } - async start() { - try { - log.debug('Starting export process for locales...', this.exportConfig.context); - log.debug(`Locales path: '${this.localesPath}'`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.localesPath); - log.debug('Created locales directory.', this.exportConfig.context); - - await this.getLocales(); - log.debug(`Retrieved ${Object.keys(this.locales).length} locales and ${Object.keys(this.masterLocale).length} master locales.`, this.exportConfig.context); - - const localesFilePath = path.join(this.localesPath, this.localeConfig.fileName); - const masterLocaleFilePath = path.join(this.localesPath, this.masterLocaleConfig.fileName); - - log.debug(`Writing locales to: '${localesFilePath}'`, this.exportConfig.context); - fsUtil.writeFile(localesFilePath, this.locales); - - log.debug(`Writing master locale to: '${masterLocaleFilePath}'`, this.exportConfig.context); - fsUtil.writeFile(masterLocaleFilePath, this.masterLocale); - - log.success( - messageHandler.parse( - 'LOCALES_EXPORT_COMPLETE', - Object.keys(this.locales).length, - Object.keys(this.masterLocale).length, - ), - this.exportConfig.context, - ); - } catch (error) { - handleAndLogError(error, { ...this.exportConfig.context }); - throw error; - } - } - - async getLocales(skip: number = 0): Promise { + async getLocales(skip = 0): Promise { if (skip) { this.qs.skip = skip; log.debug(`Fetching locales with skip: ${skip}.`, this.exportConfig.context); } log.debug(`Query parameters: ${JSON.stringify(this.qs)}.`, this.exportConfig.context); - let localesFetchResponse = await this.stackAPIClient.locale().query(this.qs).find(); + const localesFetchResponse = await this.stackAPIClient.locale().query(this.qs).find(); log.debug(`Fetched ${localesFetchResponse.items?.length || 0} locales out of ${localesFetchResponse.count}.`, this.exportConfig.context); @@ -122,7 +88,7 @@ export default class LocaleExport extends BaseClass { log.debug(`Sanitizing ${locales.length} locales...`, this.exportConfig.context); locales.forEach((locale: Record) => { - for (let key in locale) { + for (const key in locale) { if (this.localeConfig.requiredKeys.indexOf(key) === -1) { delete locale[key]; } @@ -139,4 +105,38 @@ export default class LocaleExport extends BaseClass { log.debug(`Sanitization complete. Master locales: ${Object.keys(this.masterLocale).length}, Regular locales: ${Object.keys(this.locales).length}.`, this.exportConfig.context); } + + async start() { + try { + log.debug('Starting export process for locales...', this.exportConfig.context); + log.debug(`Locales path: '${this.localesPath}'`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.localesPath); + log.debug('Created locales directory.', this.exportConfig.context); + + await this.getLocales(); + log.debug(`Retrieved ${Object.keys(this.locales).length} locales and ${Object.keys(this.masterLocale).length} master locales.`, this.exportConfig.context); + + const localesFilePath = path.join(this.localesPath, this.localeConfig.fileName); + const masterLocaleFilePath = path.join(this.localesPath, this.masterLocaleConfig.fileName); + + log.debug(`Writing locales to: '${localesFilePath}'`, this.exportConfig.context); + fsUtil.writeFile(localesFilePath, this.locales); + + log.debug(`Writing master locale to: '${masterLocaleFilePath}'`, this.exportConfig.context); + fsUtil.writeFile(masterLocaleFilePath, this.masterLocale); + + log.success( + messageHandler.parse( + 'LOCALES_EXPORT_COMPLETE', + Object.keys(this.locales).length, + Object.keys(this.masterLocale).length, + ), + this.exportConfig.context, + ); + } catch (error) { + handleAndLogError(error, { ...this.exportConfig.context }); + throw error; + } + } } diff --git a/packages/contentstack-export/src/export/modules/marketplace-apps.ts b/packages/contentstack-export/src/export/modules/marketplace-apps.ts index 094c05a22..0e08ae2e9 100644 --- a/packages/contentstack-export/src/export/modules/marketplace-apps.ts +++ b/packages/contentstack-export/src/export/modules/marketplace-apps.ts @@ -1,79 +1,42 @@ -import map from 'lodash/map'; -import has from 'lodash/has'; -import find from 'lodash/find'; -import omitBy from 'lodash/omitBy'; -import entries from 'lodash/entries'; -import isEmpty from 'lodash/isEmpty'; -import { resolve as pResolve } from 'node:path'; import { Command } from '@contentstack/cli-command'; import { - cliux, + ContentstackMarketplaceClient, NodeCrypto, + cliux, + handleAndLogError, isAuthenticated, - marketplaceSDKClient, - ContentstackMarketplaceClient, log, + marketplaceSDKClient, messageHandler, - handleAndLogError, } from '@contentstack/cli-utilities'; +import entries from 'lodash/entries'; +import find from 'lodash/find'; +import has from 'lodash/has'; +import isEmpty from 'lodash/isEmpty'; +import map from 'lodash/map'; +import omitBy from 'lodash/omitBy'; +import { resolve as pResolve } from 'node:path'; -import { fsUtil, getOrgUid, createNodeCryptoInstance, getDeveloperHubUrl } from '../../utils'; -import { ModuleClassParams, MarketplaceAppsConfig, ExportConfig, Installation, Manifest } from '../../types'; +import { ExportConfig, Installation, Manifest, MarketplaceAppsConfig, ModuleClassParams } from '../../types'; +import { createNodeCryptoInstance, fsUtil, getDeveloperHubUrl, getOrgUid } from '../../utils'; export default class ExportMarketplaceApps { - protected marketplaceAppConfig: MarketplaceAppsConfig; - protected installedApps: Installation[] = []; + public appSdk: ContentstackMarketplaceClient; + public command: Command; public developerHubBaseUrl: string; + public exportConfig: ExportConfig; + protected installedApps: Installation[] = []; + protected marketplaceAppConfig: MarketplaceAppsConfig; public marketplaceAppPath: string; public nodeCrypto: NodeCrypto; - public appSdk: ContentstackMarketplaceClient; - public exportConfig: ExportConfig; - public command: Command; public query: Record; - constructor({ exportConfig }: Omit) { + constructor({ exportConfig }: Omit) { this.exportConfig = exportConfig; this.marketplaceAppConfig = exportConfig.modules.marketplace_apps; this.exportConfig.context.module = 'marketplace-apps'; } - async start(): Promise { - log.debug('Starting export process for Marketplace Apps...', this.exportConfig.context); - - if (!isAuthenticated()) { - cliux.print( - 'WARNING!!! To export Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in', - { color: 'yellow' }, - ); - return Promise.resolve(); - } - - this.marketplaceAppPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.marketplaceAppConfig.dirName, - ); - log.debug(`Marketplace apps folder path: '${this.marketplaceAppPath}'`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.marketplaceAppPath); - log.debug('Created Marketplace Apps directory.', this.exportConfig.context); - - this.developerHubBaseUrl = this.exportConfig.developerHubBaseUrl || (await getDeveloperHubUrl(this.exportConfig)); - log.debug(`Developer Hub base URL: '${this.developerHubBaseUrl}'`, this.exportConfig.context); - - this.exportConfig.org_uid = await getOrgUid(this.exportConfig); - this.query = { target_uids: this.exportConfig.source_stack }; - log.debug(`Organization UID: '${this.exportConfig.org_uid}'.`, this.exportConfig.context); - - // NOTE init marketplace app sdk - const host = this.developerHubBaseUrl.split('://').pop(); - log.debug(`Initializing Marketplace SDK with host: '${host}'...`, this.exportConfig.context); - this.appSdk = await marketplaceSDKClient({ host }); - - await this.exportApps(); - log.debug('Marketplace apps export process completed.', this.exportConfig.context); - } - /** * The function `exportApps` encrypts the configuration of installed apps using a Node.js crypto * library if it is available. @@ -113,72 +76,6 @@ export default class ExportMarketplaceApps { log.debug(`Processed ${this.installedApps.length} Marketplace Apps.`, this.exportConfig.context); } - /** - * The function `getAppManifestAndAppConfig` exports the manifest and configurations of installed - * marketplace apps. - */ - async getAppManifestAndAppConfig(): Promise { - if (isEmpty(this.installedApps)) { - log.info(messageHandler.parse('MARKETPLACE_APPS_NOT_FOUND'), this.exportConfig.context); - } else { - log.debug(`Processing ${this.installedApps.length} installed apps...`, this.exportConfig.context); - - for (const [index, app] of entries(this.installedApps)) { - if (app.manifest.visibility === 'private') { - log.debug(`Processing private app manifest: '${app.manifest.name}'...`, this.exportConfig.context); - await this.getPrivateAppsManifest(+index, app); - } - } - - for (const [index, app] of entries(this.installedApps)) { - log.debug(`Processing app configurations for: '${app.manifest?.name || app.uid}'...`, this.exportConfig.context); - await this.getAppConfigurations(+index, app); - } - - const marketplaceAppsFilePath = pResolve(this.marketplaceAppPath, this.marketplaceAppConfig.fileName); - log.debug(`Writing Marketplace Apps to: '${marketplaceAppsFilePath}'`, this.exportConfig.context); - fsUtil.writeFile(marketplaceAppsFilePath, this.installedApps); - - log.success( - messageHandler.parse('MARKETPLACE_APPS_EXPORT_COMPLETE', Object.keys(this.installedApps).length), - this.exportConfig.context, - ); - } - } - - /** - * The function `getPrivateAppsManifest` fetches the manifest of a private app and assigns it to the - * `manifest` property of the corresponding installed app. - * @param {number} index - The `index` parameter is a number that represents the position of the app - * in an array or list. It is used to identify the specific app in the `installedApps` array. - * @param {App} appInstallation - The `appInstallation` parameter is an object that represents the - * installation details of an app. It contains information such as the UID (unique identifier) of the - * app's manifest. - */ - async getPrivateAppsManifest(index: number, appInstallation: Installation) { - log.debug(`Fetching private app manifest for: '${appInstallation.manifest.name}' (UID: ${appInstallation.manifest.uid})...`, this.exportConfig.context); - - const manifest = await this.appSdk - .marketplace(this.exportConfig.org_uid) - .app(appInstallation.manifest.uid) - .fetch({ include_oauth: true }) - .catch((error) => { - log.debug(`Failed to fetch private app manifest for: '${appInstallation.manifest.name}'.`, this.exportConfig.context); - handleAndLogError( - error, - { - ...this.exportConfig.context, - }, - messageHandler.parse('MARKETPLACE_APP_MANIFEST_EXPORT_FAILED', appInstallation.manifest.name), - ); - }); - - if (manifest) { - log.debug(`Successfully fetched private app manifest for: '${appInstallation.manifest.name}'.`, this.exportConfig.context); - this.installedApps[index].manifest = manifest as unknown as Manifest; - } - } - /** * The function `getAppConfigurations` exports the configuration of an app installation and encrypts * the server configuration if it exists. @@ -246,6 +143,72 @@ export default class ExportMarketplaceApps { }); } + /** + * The function `getAppManifestAndAppConfig` exports the manifest and configurations of installed + * marketplace apps. + */ + async getAppManifestAndAppConfig(): Promise { + if (isEmpty(this.installedApps)) { + log.info(messageHandler.parse('MARKETPLACE_APPS_NOT_FOUND'), this.exportConfig.context); + } else { + log.debug(`Processing ${this.installedApps.length} installed apps...`, this.exportConfig.context); + + for (const [index, app] of entries(this.installedApps)) { + if (app.manifest.visibility === 'private') { + log.debug(`Processing private app manifest: '${app.manifest.name}'...`, this.exportConfig.context); + await this.getPrivateAppsManifest(+index, app); + } + } + + for (const [index, app] of entries(this.installedApps)) { + log.debug(`Processing app configurations for: '${app.manifest?.name || app.uid}'...`, this.exportConfig.context); + await this.getAppConfigurations(+index, app); + } + + const marketplaceAppsFilePath = pResolve(this.marketplaceAppPath, this.marketplaceAppConfig.fileName); + log.debug(`Writing Marketplace Apps to: '${marketplaceAppsFilePath}'`, this.exportConfig.context); + fsUtil.writeFile(marketplaceAppsFilePath, this.installedApps); + + log.success( + messageHandler.parse('MARKETPLACE_APPS_EXPORT_COMPLETE', Object.keys(this.installedApps).length), + this.exportConfig.context, + ); + } + } + + /** + * The function `getPrivateAppsManifest` fetches the manifest of a private app and assigns it to the + * `manifest` property of the corresponding installed app. + * @param {number} index - The `index` parameter is a number that represents the position of the app + * in an array or list. It is used to identify the specific app in the `installedApps` array. + * @param {App} appInstallation - The `appInstallation` parameter is an object that represents the + * installation details of an app. It contains information such as the UID (unique identifier) of the + * app's manifest. + */ + async getPrivateAppsManifest(index: number, appInstallation: Installation) { + log.debug(`Fetching private app manifest for: '${appInstallation.manifest.name}' (UID: ${appInstallation.manifest.uid})...`, this.exportConfig.context); + + const manifest = await this.appSdk + .marketplace(this.exportConfig.org_uid) + .app(appInstallation.manifest.uid) + .fetch({ include_oauth: true }) + .catch((error) => { + log.debug(`Failed to fetch private app manifest for: '${appInstallation.manifest.name}'.`, this.exportConfig.context); + handleAndLogError( + error, + { + ...this.exportConfig.context, + }, + messageHandler.parse('MARKETPLACE_APP_MANIFEST_EXPORT_FAILED', appInstallation.manifest.name), + ); + }); + + if (manifest) { + log.debug(`Successfully fetched private app manifest for: '${appInstallation.manifest.name}'.`, this.exportConfig.context); + this.installedApps[index].manifest = manifest as unknown as Manifest; + } + } + /** * The function `getStackSpecificApps` retrieves a collection of marketplace apps specific to a stack * and stores them in the `installedApps` array. @@ -267,7 +230,7 @@ export default class ExportMarketplaceApps { }); if (collection) { - const { items: apps, count } = collection; + const { count, items: apps } = collection; log.debug(`Fetched ${apps?.length || 0} apps out of ${count}.`, this.exportConfig.context); // NOTE Remove all the chain functions @@ -289,4 +252,41 @@ export default class ExportMarketplaceApps { } } } + + async start(): Promise { + log.debug('Starting export process for Marketplace Apps...', this.exportConfig.context); + + if (!isAuthenticated()) { + cliux.print( + 'WARNING!!! To export Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in', + { color: 'yellow' }, + ); + return Promise.resolve(); + } + + this.marketplaceAppPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.marketplaceAppConfig.dirName, + ); + log.debug(`Marketplace apps folder path: '${this.marketplaceAppPath}'`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.marketplaceAppPath); + log.debug('Created Marketplace Apps directory.', this.exportConfig.context); + + this.developerHubBaseUrl = this.exportConfig.developerHubBaseUrl || (await getDeveloperHubUrl(this.exportConfig)); + log.debug(`Developer Hub base URL: '${this.developerHubBaseUrl}'`, this.exportConfig.context); + + this.exportConfig.org_uid = await getOrgUid(this.exportConfig); + this.query = { target_uids: this.exportConfig.source_stack }; + log.debug(`Organization UID: '${this.exportConfig.org_uid}'.`, this.exportConfig.context); + + // NOTE init marketplace app sdk + const host = this.developerHubBaseUrl.split('://').pop(); + log.debug(`Initializing Marketplace SDK with host: '${host}'...`, this.exportConfig.context); + this.appSdk = await marketplaceSDKClient({ host }); + + await this.exportApps(); + log.debug('Marketplace apps export process completed.', this.exportConfig.context); + } } diff --git a/packages/contentstack-export/src/export/modules/personalize.ts b/packages/contentstack-export/src/export/modules/personalize.ts index 51656635c..e154321c5 100644 --- a/packages/contentstack-export/src/export/modules/personalize.ts +++ b/packages/contentstack-export/src/export/modules/personalize.ts @@ -1,18 +1,18 @@ +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import { - ExportProjects, - ExportExperiences, - ExportEvents, + AnyProperty, ExportAttributes, ExportAudiences, - AnyProperty, + ExportEvents, + ExportExperiences, + ExportProjects, } from '@contentstack/cli-variants'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; -import { ModuleClassParams, ExportConfig } from '../../types'; +import { ExportConfig, ModuleClassParams } from '../../types'; export default class ExportPersonalize { public exportConfig: ExportConfig; - public personalizeConfig: { dirName: string; baseURL: Record } & AnyProperty; + public personalizeConfig: { baseURL: Record; dirName: string } & AnyProperty; constructor({ exportConfig }: ModuleClassParams) { this.exportConfig = exportConfig; this.personalizeConfig = exportConfig.modules.personalize; @@ -44,9 +44,9 @@ export default class ExportPersonalize { log.debug('Personalization is enabled, processing personalize modules... ' + this.exportConfig.modules.personalize.exportOrder.join(', '), this.exportConfig.context); const moduleMapper = { - events: new ExportEvents(this.exportConfig), attributes: new ExportAttributes(this.exportConfig), audiences: new ExportAudiences(this.exportConfig), + events: new ExportEvents(this.exportConfig), experiences: new ExportExperiences(this.exportConfig), }; diff --git a/packages/contentstack-export/src/export/modules/publishing-rules.ts b/packages/contentstack-export/src/export/modules/publishing-rules.ts new file mode 100644 index 000000000..fee752b10 --- /dev/null +++ b/packages/contentstack-export/src/export/modules/publishing-rules.ts @@ -0,0 +1,86 @@ +import { handleAndLogError, log } from '@contentstack/cli-utilities'; +import isEmpty from 'lodash/isEmpty'; +import omit from 'lodash/omit'; +import { resolve as pResolve } from 'node:path'; + +import { ModuleClassParams, PublishingRulesConfig } from '../../types'; +import { fsUtil } from '../../utils'; +import BaseClass from './base-class'; + +export default class ExportPublishingRules extends BaseClass { + private readonly publishingRules: Record> = {}; + private readonly publishingRulesConfig: PublishingRulesConfig; + private publishingRulesFolderPath: string; + private readonly qs: { include_count: boolean; skip?: number }; + + constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { + super({ exportConfig, stackAPIClient }); + this.publishingRulesConfig = exportConfig.modules['publishing-rules']; + this.qs = { include_count: true }; + this.exportConfig.context.module = 'publishing-rules'; + } + + async start(): Promise { + this.publishingRulesFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.publishingRulesConfig.dirName, + ); + log.debug(`Publishing rules folder path: ${this.publishingRulesFolderPath}`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.publishingRulesFolderPath); + log.debug('Created publishing rules directory', this.exportConfig.context); + + await this.fetchAllPublishingRules(); + + if (isEmpty(this.publishingRules)) { + log.info('No Publishing Rules found', this.exportConfig.context); + return; + } + + const outPath = pResolve(this.publishingRulesFolderPath, this.publishingRulesConfig.fileName); + fsUtil.writeFile(outPath, this.publishingRules); + log.success( + `Publishing rules exported successfully! Total count: ${Object.keys(this.publishingRules).length}`, + this.exportConfig.context, + ); + } + + private async fetchAllPublishingRules(skip = 0): Promise { + try { + if (skip > 0) { + this.qs.skip = skip; + } + + const data: { count?: number; items?: Record[] } = await this.stack + .workflow() + .publishRule() + .fetchAll(this.qs); + + const items = data.items ?? []; + const total = data.count ?? items.length; + + if (!items.length) { + log.debug('No publishing rules returned for this page', this.exportConfig.context); + return; + } + + for (const rule of items) { + const uid = rule.uid as string | undefined; + if (uid) { + this.publishingRules[uid] = omit(rule, this.publishingRulesConfig.invalidKeys) as Record< + string, + unknown + >; + } + } + + const nextSkip = skip + items.length; + if (nextSkip < total) { + await this.fetchAllPublishingRules(nextSkip); + } + } catch (error: unknown) { + handleAndLogError(error as Error, { ...this.exportConfig.context }); + } + } +} diff --git a/packages/contentstack-export/src/export/modules/stack.ts b/packages/contentstack-export/src/export/modules/stack.ts index cb1d75fb0..92ee1f9b3 100644 --- a/packages/contentstack-export/src/export/modules/stack.ts +++ b/packages/contentstack-export/src/export/modules/stack.ts @@ -1,18 +1,18 @@ +import { handleAndLogError, isAuthenticated, log, managementSDKClient } from '@contentstack/cli-utilities'; import find from 'lodash/find'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, isAuthenticated, managementSDKClient, log } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; +import { ModuleClassParams, StackConfig } from '../../types'; import { fsUtil } from '../../utils'; -import { StackConfig, ModuleClassParams } from '../../types'; +import BaseClass from './base-class'; export default class ExportStack extends BaseClass { - private stackConfig: StackConfig; - private stackFolderPath: string; private qs: { include_count: boolean; skip?: number; }; + private stackConfig: StackConfig; + private stackFolderPath: string; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -26,67 +26,47 @@ export default class ExportStack extends BaseClass { this.exportConfig.context.module = 'stack'; } - async start(): Promise { - log.debug('Starting stack export process...', this.exportConfig.context); + async exportStack(): Promise { + log.debug(`Starting stack export for: '${this.exportConfig.source_stack}'...`, this.exportConfig.context); - if (isAuthenticated()) { - log.debug('User authenticated.', this.exportConfig.context); - const stackData = await this.getStack(); - if (stackData?.org_uid) { - log.debug(`Found organization UID: '${stackData.org_uid}'.`, this.exportConfig.context); - this.exportConfig.org_uid = stackData.org_uid; - this.exportConfig.sourceStackName = stackData.name; - log.debug(`Set source stack name: '${stackData.name}'.`, this.exportConfig.context); - } else { - log.debug('No stack data found or missing organization UID.', this.exportConfig.context); - } - } else { - log.debug('User is not authenticated.', this.exportConfig.context); - } + await fsUtil.makeDirectory(this.stackFolderPath); + log.debug(`Created stack directory at: '${this.stackFolderPath}'`, this.exportConfig.context); - if (this.exportConfig.management_token) { - log.info( - 'Skipping stack settings export: Operation is not supported when using a management token.', - this.exportConfig.context, - ); - } else { - await this.exportStackSettings(); - } - if (!this.exportConfig.preserveStackVersion && !this.exportConfig.hasOwnProperty('master_locale')) { - log.debug( - 'Preserve stack version is false and master locale not set, fetching locales...', - this.exportConfig.context, - ); - //fetch master locale details - return this.getLocales(); - } else if (this.exportConfig.preserveStackVersion) { - log.debug('Preserve stack version is set to true.', this.exportConfig.context); - return this.exportStack(); - } else { - log.debug('Master locale is already set.', this.exportConfig.context); - } + return this.stack + .fetch() + .then((resp: any) => { + const stackFilePath = pResolve(this.stackFolderPath, this.stackConfig.fileName); + log.debug(`Writing stack data to: '${stackFilePath}'`, this.exportConfig.context); + fsUtil.writeFile(stackFilePath, resp); + log.success( + `Stack details exported successfully for stack ${this.exportConfig.source_stack}`, + this.exportConfig.context, + ); + log.debug('Stack export completed successfully.', this.exportConfig.context); + return resp; + }) + .catch((error: any) => { + log.debug(`An error occurred while exporting stack: '${this.exportConfig.source_stack}'.`, this.exportConfig.context); + handleAndLogError(error, { ...this.exportConfig.context }); + }); } - async getStack(): Promise { - log.debug(`Fetching stack data for: '${this.exportConfig.source_stack}'...`, this.exportConfig.context); - - const tempAPIClient = await managementSDKClient({ host: this.exportConfig.host }); - log.debug(`Created Management SDK client with host: '${this.exportConfig.host}'.`, this.exportConfig.context); - - return await tempAPIClient - .stack({ api_key: this.exportConfig.source_stack }) - .fetch() - .then((data: any) => { - log.debug(`Successfully fetched stack data for: '${this.exportConfig.source_stack}'.`, this.exportConfig.context); - return data; + async exportStackSettings(): Promise { + log.info('Exporting stack settings...', this.exportConfig.context); + await fsUtil.makeDirectory(this.stackFolderPath); + return this.stack + .settings() + .then((resp: any) => { + fsUtil.writeFile(pResolve(this.stackFolderPath, 'settings.json'), resp); + log.success('Exported stack settings successfully!', this.exportConfig.context); + return resp; }) .catch((error: any) => { - log.debug(`Failed to fetch stack data for: '${this.exportConfig.source_stack}'.`, this.exportConfig.context); - return {}; + handleAndLogError(error, { ...this.exportConfig.context }); }); } - async getLocales(skip: number = 0) { + async getLocales(skip = 0) { if (skip) { this.qs.skip = skip; log.debug(`Fetching locales with skip: ${skip}.`, this.exportConfig.context); @@ -101,7 +81,7 @@ export default class ExportStack extends BaseClass { .query(this.qs) .find() .then(async (data: any) => { - const { items, count } = data; + const { count, items } = data; log.debug(`Fetched ${items?.length || 0} locales out of ${count}.`, this.exportConfig.context); if (items?.length) { @@ -148,43 +128,63 @@ export default class ExportStack extends BaseClass { }); } - async exportStack(): Promise { - log.debug(`Starting stack export for: '${this.exportConfig.source_stack}'...`, this.exportConfig.context); + async getStack(): Promise { + log.debug(`Fetching stack data for: '${this.exportConfig.source_stack}'...`, this.exportConfig.context); - await fsUtil.makeDirectory(this.stackFolderPath); - log.debug(`Created stack directory at: '${this.stackFolderPath}'`, this.exportConfig.context); + const tempAPIClient = await managementSDKClient({ host: this.exportConfig.host }); + log.debug(`Created Management SDK client with host: '${this.exportConfig.host}'.`, this.exportConfig.context); - return this.stack + return await tempAPIClient + .stack({ api_key: this.exportConfig.source_stack }) .fetch() - .then((resp: any) => { - const stackFilePath = pResolve(this.stackFolderPath, this.stackConfig.fileName); - log.debug(`Writing stack data to: '${stackFilePath}'`, this.exportConfig.context); - fsUtil.writeFile(stackFilePath, resp); - log.success( - `Stack details exported successfully for stack ${this.exportConfig.source_stack}`, - this.exportConfig.context, - ); - log.debug('Stack export completed successfully.', this.exportConfig.context); - return resp; + .then((data: any) => { + log.debug(`Successfully fetched stack data for: '${this.exportConfig.source_stack}'.`, this.exportConfig.context); + return data; }) .catch((error: any) => { - log.debug(`An error occurred while exporting stack: '${this.exportConfig.source_stack}'.`, this.exportConfig.context); - handleAndLogError(error, { ...this.exportConfig.context }); + log.debug(`Failed to fetch stack data for: '${this.exportConfig.source_stack}'.`, this.exportConfig.context); + return {}; }); } - async exportStackSettings(): Promise { - log.info('Exporting stack settings...', this.exportConfig.context); - await fsUtil.makeDirectory(this.stackFolderPath); - return this.stack - .settings() - .then((resp: any) => { - fsUtil.writeFile(pResolve(this.stackFolderPath, 'settings.json'), resp); - log.success('Exported stack settings successfully!', this.exportConfig.context); - return resp; - }) - .catch((error: any) => { - handleAndLogError(error, { ...this.exportConfig.context }); - }); + async start(): Promise { + log.debug('Starting stack export process...', this.exportConfig.context); + + if (isAuthenticated()) { + log.debug('User authenticated.', this.exportConfig.context); + const stackData = await this.getStack(); + if (stackData?.org_uid) { + log.debug(`Found organization UID: '${stackData.org_uid}'.`, this.exportConfig.context); + this.exportConfig.org_uid = stackData.org_uid; + this.exportConfig.sourceStackName = stackData.name; + log.debug(`Set source stack name: '${stackData.name}'.`, this.exportConfig.context); + } else { + log.debug('No stack data found or missing organization UID.', this.exportConfig.context); + } + } else { + log.debug('User is not authenticated.', this.exportConfig.context); + } + + if (this.exportConfig.management_token) { + log.info( + 'Skipping stack settings export: Operation is not supported when using a management token.', + this.exportConfig.context, + ); + } else { + await this.exportStackSettings(); + } + if (!this.exportConfig.preserveStackVersion && !this.exportConfig.hasOwnProperty('master_locale')) { + log.debug( + 'Preserve stack version is false and master locale not set, fetching locales...', + this.exportConfig.context, + ); + //fetch master locale details + return this.getLocales(); + } else if (this.exportConfig.preserveStackVersion) { + log.debug('Preserve stack version is set to true.', this.exportConfig.context); + return this.exportStack(); + } else { + log.debug('Master locale is already set.', this.exportConfig.context); + } } } diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 69acea863..ae8cbc287 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,30 +1,30 @@ -import omit from 'lodash/omit'; -import keys from 'lodash/keys'; +import { handleAndLogError, log, messageHandler, sanitizePath } from '@contentstack/cli-utilities'; import isEmpty from 'lodash/isEmpty'; +import keys from 'lodash/keys'; +import omit from 'lodash/omit'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, messageHandler, log, sanitizePath } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; +import { ExportConfig, ModuleClassParams } from '../../types'; import { fsUtil } from '../../utils'; -import { ModuleClassParams, ExportConfig } from '../../types'; +import BaseClass from './base-class'; export default class ExportTaxonomies extends BaseClass { - private taxonomies: Record>; - private taxonomiesByLocale: Record>; - private taxonomiesConfig: ExportConfig['modules']['taxonomies']; - private isLocaleBasedExportSupported: boolean = true; // Flag to track if locale-based export is supported + public taxonomiesFolderPath: string; + private isLocaleBasedExportSupported = true; + private localesFilePath: string; private qs: { - include_count: boolean; - skip: number; asc?: string; - limit: number; - locale?: string; branch?: string; - include_fallback?: boolean; fallback_locale?: string; - }; - public taxonomiesFolderPath: string; - private localesFilePath: string; + include_count: boolean; + include_fallback?: boolean; + limit: number; + locale?: string; + skip: number; + }; // Flag to track if locale-based export is supported + private taxonomies: Record>; + private taxonomiesByLocale: Record>; + private taxonomiesConfig: ExportConfig['modules']['taxonomies']; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -43,87 +43,57 @@ export default class ExportTaxonomies extends BaseClass { ); } - async start(): Promise { - log.debug('Starting export process for taxonomies...', this.exportConfig.context); - - //create taxonomies folder - this.taxonomiesFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.taxonomiesConfig.dirName, - ); - log.debug(`Taxonomies folder path: '${this.taxonomiesFolderPath}'`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.taxonomiesFolderPath); - log.debug('Created taxonomies directory.', this.exportConfig.context); - - const localesToExport = this.getLocalesToExport(); - log.debug( - `Will attempt to export taxonomies for ${localesToExport.length} locale(s): ${localesToExport.join(', ')}`, - this.exportConfig.context, - ); + /** + * Export taxonomies - supports both locale-based and legacy export + */ + async exportTaxonomies(localeCode?: string): Promise { + const taxonomiesUID = localeCode ? Array.from(this.taxonomiesByLocale[localeCode] || []) : keys(this.taxonomies); - if (localesToExport.length === 0) { - log.warn('No locales found to export', this.exportConfig.context); + const localeInfo = localeCode ? ` for locale: ${localeCode}` : ''; + if (taxonomiesUID.length === 0) { + log.debug(`No taxonomies to export${localeInfo}`, this.exportConfig.context); return; } + log.debug(`Exporting detailed data for ${taxonomiesUID.length} taxonomies${localeInfo}`, this.exportConfig.context); - // Test locale-based export support with master locale - const masterLocale = this.exportConfig.master_locale?.code; - await this.fetchTaxonomies(masterLocale, true); - - if (!this.isLocaleBasedExportSupported) { - this.taxonomies = {}; - this.taxonomiesByLocale = {}; - - // Fetch taxonomies without locale parameter - await this.fetchTaxonomies(); - await this.exportTaxonomies(); - await this.writeTaxonomiesMetadata(); - } else { - // Process all locales with locale-based export - log.debug('Localization enabled, proceeding with locale-based export', this.exportConfig.context); - - for (const localeCode of localesToExport) { - await this.fetchTaxonomies(localeCode); - await this.processLocaleExport(localeCode); - } - - await this.writeTaxonomiesMetadata(); + const exportFolderPath = localeCode ? pResolve(this.taxonomiesFolderPath, localeCode) : this.taxonomiesFolderPath; + if (localeCode) { + await fsUtil.makeDirectory(exportFolderPath); + log.debug(`Created locale folder: ${exportFolderPath}`, this.exportConfig.context); } - log.success( - messageHandler.parse('TAXONOMY_EXPORT_COMPLETE', keys(this.taxonomies || {}).length), - this.exportConfig.context, - ); - } + const onSuccess = ({ response, uid }: any) => { + const filePath = pResolve(exportFolderPath, `${uid}.json`); + log.debug(`Writing detailed taxonomy data to: ${filePath}`, this.exportConfig.context); + fsUtil.writeFile(filePath, response); + log.success(messageHandler.parse('TAXONOMY_EXPORT_SUCCESS', uid), this.exportConfig.context); + }; - /** - * Process and export taxonomies for a specific locale - */ - async processLocaleExport(localeCode: string): Promise { - const localeTaxonomies = this.taxonomiesByLocale[localeCode]; + const onReject = ({ error, uid }: any) => { + log.debug(`Failed to export detailed data for taxonomy: ${uid}${localeInfo}`, this.exportConfig.context); + handleAndLogError(error, { ...this.exportConfig.context, uid, ...(localeCode && { locale: localeCode }) }); + }; - if (localeTaxonomies?.size > 0) { - log.info(`Found ${localeTaxonomies.size} taxonomies for locale: ${localeCode}`, this.exportConfig.context); - await this.exportTaxonomies(localeCode); - } else { - log.debug(`No taxonomies found for locale: ${localeCode}`, this.exportConfig.context); - } - } + for (const taxonomyUID of taxonomiesUID) { + log.debug(`Processing detailed export for taxonomy: ${taxonomyUID}${localeInfo}`, this.exportConfig.context); - /** - * Write taxonomies metadata file - */ - async writeTaxonomiesMetadata(): Promise { - if (!this.taxonomies || isEmpty(this.taxonomies)) { - log.info(messageHandler.parse('TAXONOMY_NOT_FOUND'), this.exportConfig.context); - return; - } + const exportParams: any = { format: 'json' }; + if (localeCode) { + exportParams.locale = localeCode; + if (this.qs.include_fallback !== undefined) exportParams.include_fallback = this.qs.include_fallback; + if (this.qs.fallback_locale) exportParams.fallback_locale = this.qs.fallback_locale; + } + if (this.qs.branch) exportParams.branch = this.qs.branch; - const taxonomiesFilePath = pResolve(this.taxonomiesFolderPath, 'taxonomies.json'); - log.debug(`Writing taxonomies metadata to: ${taxonomiesFilePath}`, this.exportConfig.context); - fsUtil.writeFile(taxonomiesFilePath, this.taxonomies); + await this.makeAPICall({ + module: 'export-taxonomy', + queryParam: exportParams, + reject: onReject, + resolve: onSuccess, + uid: taxonomyUID, + }); + } + log.debug(`Completed detailed taxonomy export process${localeInfo}`, this.exportConfig.context); } /** @@ -134,7 +104,7 @@ export default class ExportTaxonomies extends BaseClass { * @param {boolean} [checkLocaleSupport=false] * @returns {Promise} */ - async fetchTaxonomies(localeCode?: string, checkLocaleSupport: boolean = false): Promise { + async fetchTaxonomies(localeCode?: string, checkLocaleSupport = false): Promise { let skip = 0; const localeInfo = localeCode ? `for locale: ${localeCode}` : ''; @@ -152,7 +122,7 @@ export default class ExportTaxonomies extends BaseClass { try { const data = await this.stack.taxonomy().query(queryParams).find(); - const { items, count } = data; + const { count, items } = data; const taxonomiesCount = count ?? items?.length ?? 0; log.debug( @@ -204,6 +174,58 @@ export default class ExportTaxonomies extends BaseClass { } while (true); } + /** + * Get all locales to export + */ + getLocalesToExport(): string[] { + log.debug('Determining locales to export...', this.exportConfig.context); + + const masterLocaleCode = this.exportConfig.master_locale?.code || 'en-us'; + const localeSet = new Set([masterLocaleCode]); + + try { + const locales = fsUtil.readFile(this.localesFilePath) as Record>; + + if (locales && keys(locales || {}).length > 0) { + log.debug( + `Loaded ${keys(locales || {}).length} locales from ${this.localesFilePath}`, + this.exportConfig.context, + ); + + for (const localeUid of keys(locales)) { + const localeCode = locales[localeUid].code; + if (localeCode && !localeSet.has(localeCode)) { + localeSet.add(localeCode); + log.debug(`Added locale: ${localeCode} (uid: ${localeUid})`, this.exportConfig.context); + } + } + } else { + log.debug(`No locales found in ${this.localesFilePath}`, this.exportConfig.context); + } + } catch (error) { + log.warn(`Failed to read locales file: ${this.localesFilePath}`, this.exportConfig.context); + } + + const localesToExport = Array.from(localeSet); + log.debug(`Total unique locales to export: ${localesToExport.length}`, this.exportConfig.context); + + return localesToExport; + } + + /** + * Process and export taxonomies for a specific locale + */ + async processLocaleExport(localeCode: string): Promise { + const localeTaxonomies = this.taxonomiesByLocale[localeCode]; + + if (localeTaxonomies?.size > 0) { + log.info(`Found ${localeTaxonomies.size} taxonomies for locale: ${localeCode}`, this.exportConfig.context); + await this.exportTaxonomies(localeCode); + } else { + log.debug(`No taxonomies found for locale: ${localeCode}`, this.exportConfig.context); + } + } + /** * remove invalid keys and write data into taxonomies * @function sanitizeTaxonomiesAttribs @@ -237,95 +259,73 @@ export default class ExportTaxonomies extends BaseClass { ); } - /** - * Export taxonomies - supports both locale-based and legacy export - */ - async exportTaxonomies(localeCode?: string): Promise { - const taxonomiesUID = localeCode ? Array.from(this.taxonomiesByLocale[localeCode] || []) : keys(this.taxonomies); + async start(): Promise { + log.debug('Starting export process for taxonomies...', this.exportConfig.context); + + //create taxonomies folder + this.taxonomiesFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.taxonomiesConfig.dirName, + ); + log.debug(`Taxonomies folder path: '${this.taxonomiesFolderPath}'`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.taxonomiesFolderPath); + log.debug('Created taxonomies directory.', this.exportConfig.context); - const localeInfo = localeCode ? ` for locale: ${localeCode}` : ''; - if (taxonomiesUID.length === 0) { - log.debug(`No taxonomies to export${localeInfo}`, this.exportConfig.context); - return; - } - log.debug(`Exporting detailed data for ${taxonomiesUID.length} taxonomies${localeInfo}`, this.exportConfig.context); + const localesToExport = this.getLocalesToExport(); + log.debug( + `Will attempt to export taxonomies for ${localesToExport.length} locale(s): ${localesToExport.join(', ')}`, + this.exportConfig.context, + ); - const exportFolderPath = localeCode ? pResolve(this.taxonomiesFolderPath, localeCode) : this.taxonomiesFolderPath; - if (localeCode) { - await fsUtil.makeDirectory(exportFolderPath); - log.debug(`Created locale folder: ${exportFolderPath}`, this.exportConfig.context); + if (localesToExport.length === 0) { + log.warn('No locales found to export', this.exportConfig.context); + return; } - const onSuccess = ({ response, uid }: any) => { - const filePath = pResolve(exportFolderPath, `${uid}.json`); - log.debug(`Writing detailed taxonomy data to: ${filePath}`, this.exportConfig.context); - fsUtil.writeFile(filePath, response); - log.success(messageHandler.parse('TAXONOMY_EXPORT_SUCCESS', uid), this.exportConfig.context); - }; + // Test locale-based export support with master locale + const masterLocale = this.exportConfig.master_locale?.code; + await this.fetchTaxonomies(masterLocale, true); - const onReject = ({ error, uid }: any) => { - log.debug(`Failed to export detailed data for taxonomy: ${uid}${localeInfo}`, this.exportConfig.context); - handleAndLogError(error, { ...this.exportConfig.context, uid, ...(localeCode && { locale: localeCode }) }); - }; + if (!this.isLocaleBasedExportSupported) { + this.taxonomies = {}; + this.taxonomiesByLocale = {}; - for (const taxonomyUID of taxonomiesUID) { - log.debug(`Processing detailed export for taxonomy: ${taxonomyUID}${localeInfo}`, this.exportConfig.context); + // Fetch taxonomies without locale parameter + await this.fetchTaxonomies(); + await this.exportTaxonomies(); + await this.writeTaxonomiesMetadata(); + } else { + // Process all locales with locale-based export + log.debug('Localization enabled, proceeding with locale-based export', this.exportConfig.context); - const exportParams: any = { format: 'json' }; - if (localeCode) { - exportParams.locale = localeCode; - if (this.qs.include_fallback !== undefined) exportParams.include_fallback = this.qs.include_fallback; - if (this.qs.fallback_locale) exportParams.fallback_locale = this.qs.fallback_locale; + for (const localeCode of localesToExport) { + await this.fetchTaxonomies(localeCode); + await this.processLocaleExport(localeCode); } - if (this.qs.branch) exportParams.branch = this.qs.branch; - await this.makeAPICall({ - reject: onReject, - resolve: onSuccess, - uid: taxonomyUID, - module: 'export-taxonomy', - queryParam: exportParams, - }); + await this.writeTaxonomiesMetadata(); } - log.debug(`Completed detailed taxonomy export process${localeInfo}`, this.exportConfig.context); + + log.success( + messageHandler.parse('TAXONOMY_EXPORT_COMPLETE', keys(this.taxonomies || {}).length), + this.exportConfig.context, + ); } /** - * Get all locales to export + * Write taxonomies metadata file */ - getLocalesToExport(): string[] { - log.debug('Determining locales to export...', this.exportConfig.context); - - const masterLocaleCode = this.exportConfig.master_locale?.code || 'en-us'; - const localeSet = new Set([masterLocaleCode]); - - try { - const locales = fsUtil.readFile(this.localesFilePath) as Record>; - - if (locales && keys(locales || {}).length > 0) { - log.debug( - `Loaded ${keys(locales || {}).length} locales from ${this.localesFilePath}`, - this.exportConfig.context, - ); - - for (const localeUid of keys(locales)) { - const localeCode = locales[localeUid].code; - if (localeCode && !localeSet.has(localeCode)) { - localeSet.add(localeCode); - log.debug(`Added locale: ${localeCode} (uid: ${localeUid})`, this.exportConfig.context); - } - } - } else { - log.debug(`No locales found in ${this.localesFilePath}`, this.exportConfig.context); - } - } catch (error) { - log.warn(`Failed to read locales file: ${this.localesFilePath}`, this.exportConfig.context); + async writeTaxonomiesMetadata(): Promise { + if (!this.taxonomies || isEmpty(this.taxonomies)) { + log.info(messageHandler.parse('TAXONOMY_NOT_FOUND'), this.exportConfig.context); + return; } - const localesToExport = Array.from(localeSet); - log.debug(`Total unique locales to export: ${localesToExport.length}`, this.exportConfig.context); - - return localesToExport; + const taxonomiesFilePath = pResolve(this.taxonomiesFolderPath, 'taxonomies.json'); + log.debug(`Writing taxonomies metadata to: ${taxonomiesFilePath}`, this.exportConfig.context); + fsUtil.writeFile(taxonomiesFilePath, this.taxonomies); } private isLocalePlanLimitationError(error: any): boolean { diff --git a/packages/contentstack-export/src/export/modules/webhooks.ts b/packages/contentstack-export/src/export/modules/webhooks.ts index 42d657490..f4d797230 100644 --- a/packages/contentstack-export/src/export/modules/webhooks.ts +++ b/packages/contentstack-export/src/export/modules/webhooks.ts @@ -1,59 +1,30 @@ -import omit from 'lodash/omit'; +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import isEmpty from 'lodash/isEmpty'; +import omit from 'lodash/omit'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; +import { ModuleClassParams, WebhookConfig } from '../../types'; import { fsUtil } from '../../utils'; -import { WebhookConfig, ModuleClassParams } from '../../types'; +import BaseClass from './base-class'; export default class ExportWebhooks extends BaseClass { - private webhooks: Record>; - private webhookConfig: WebhookConfig; public webhooksFolderPath: string; private qs: { + asc: string; include_count: boolean; skip?: number; - asc: string; }; + private webhookConfig: WebhookConfig; + private webhooks: Record>; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); this.webhooks = {}; this.webhookConfig = exportConfig.modules.webhooks; - this.qs = { include_count: true, asc: 'updated_at' }; + this.qs = { asc: 'updated_at', include_count: true }; this.exportConfig.context.module = 'webhooks'; } - async start(): Promise { - log.debug('Starting webhooks export process...', this.exportConfig.context); - - this.webhooksFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.webhookConfig.dirName, - ); - log.debug(`Webhooks folder path: ${this.webhooksFolderPath}`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.webhooksFolderPath); - log.debug('Created webhooks directory', this.exportConfig.context); - - await this.getWebhooks(); - log.debug(`Retrieved ${Object.keys(this.webhooks).length} webhooks`, this.exportConfig.context); - - if (this.webhooks === undefined || isEmpty(this.webhooks)) { - log.info(messageHandler.parse('WEBHOOK_NOT_FOUND'), this.exportConfig.context); - } else { - const webhooksFilePath = pResolve(this.webhooksFolderPath, this.webhookConfig.fileName); - log.debug(`Writing webhooks to: ${webhooksFilePath}`, this.exportConfig.context); - fsUtil.writeFile(webhooksFilePath, this.webhooks); - log.success( - messageHandler.parse('WEBHOOK_EXPORT_COMPLETE', Object.keys(this.webhooks).length), - this.exportConfig.context, - ); - } - } - async getWebhooks(skip = 0): Promise { if (skip) { this.qs.skip = skip; @@ -68,7 +39,7 @@ export default class ExportWebhooks extends BaseClass { .webhook() .fetchAll(this.qs) .then(async (data: any) => { - const { items, count } = data; + const { count, items } = data; log.debug(`Fetched ${items?.length || 0} webhooks out of total ${count}`, this.exportConfig.context); if (items?.length) { @@ -105,4 +76,33 @@ export default class ExportWebhooks extends BaseClass { log.debug(`Sanitization complete. Total webhooks processed: ${Object.keys(this.webhooks).length}`, this.exportConfig.context); } + + async start(): Promise { + log.debug('Starting webhooks export process...', this.exportConfig.context); + + this.webhooksFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.webhookConfig.dirName, + ); + log.debug(`Webhooks folder path: ${this.webhooksFolderPath}`, this.exportConfig.context); + + await fsUtil.makeDirectory(this.webhooksFolderPath); + log.debug('Created webhooks directory', this.exportConfig.context); + + await this.getWebhooks(); + log.debug(`Retrieved ${Object.keys(this.webhooks).length} webhooks`, this.exportConfig.context); + + if (this.webhooks === undefined || isEmpty(this.webhooks)) { + log.info(messageHandler.parse('WEBHOOK_NOT_FOUND'), this.exportConfig.context); + } else { + const webhooksFilePath = pResolve(this.webhooksFolderPath, this.webhookConfig.fileName); + log.debug(`Writing webhooks to: ${webhooksFilePath}`, this.exportConfig.context); + fsUtil.writeFile(webhooksFilePath, this.webhooks); + log.success( + messageHandler.parse('WEBHOOK_EXPORT_COMPLETE', Object.keys(this.webhooks).length), + this.exportConfig.context, + ); + } + } } diff --git a/packages/contentstack-export/src/export/modules/workflows.ts b/packages/contentstack-export/src/export/modules/workflows.ts index 6a9fedb8a..d3aacf3a8 100644 --- a/packages/contentstack-export/src/export/modules/workflows.ts +++ b/packages/contentstack-export/src/export/modules/workflows.ts @@ -1,20 +1,20 @@ -import omit from 'lodash/omit'; +import { handleAndLogError, log, messageHandler } from '@contentstack/cli-utilities'; import isEmpty from 'lodash/isEmpty'; +import omit from 'lodash/omit'; import { resolve as pResolve } from 'node:path'; -import { handleAndLogError, messageHandler, log } from '@contentstack/cli-utilities'; -import BaseClass from './base-class'; +import { ModuleClassParams, WorkflowConfig } from '../../types'; import { fsUtil } from '../../utils'; -import { WorkflowConfig, ModuleClassParams } from '../../types'; +import BaseClass from './base-class'; export default class ExportWorkFlows extends BaseClass { - private workflows: Record>; - private workflowConfig: WorkflowConfig; public webhooksFolderPath: string; private qs: { include_count: boolean; skip?: number; }; + private workflowConfig: WorkflowConfig; + private workflows: Record>; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -24,30 +24,37 @@ export default class ExportWorkFlows extends BaseClass { this.exportConfig.context.module = 'workflows'; } - async start(): Promise { - this.webhooksFolderPath = pResolve( - this.exportConfig.data, - this.exportConfig.branchName || '', - this.workflowConfig.dirName, - ); - log.debug(`Workflows folder path: ${this.webhooksFolderPath}`, this.exportConfig.context); - - await fsUtil.makeDirectory(this.webhooksFolderPath); - log.debug('Created workflows directory', this.exportConfig.context); + async getRoles(roleUid: number): Promise { + log.debug(`Fetching role with UID: ${roleUid}`, this.exportConfig.context); - await this.getWorkflows(); - log.debug(`Retrieved ${Object.keys(this.workflows).length} workflows`, this.exportConfig.context); + return await this.stack + .role(roleUid) + .fetch({ include_permissions: true, include_rules: true }) + .then((data: any) => { + log.debug(`Successfully fetched role data for UID: ${roleUid}`, this.exportConfig.context); + return data; + }) + .catch((err: any) => { + log.debug(`Failed to fetch role data for UID: ${roleUid}`, this.exportConfig.context); + handleAndLogError( + err, + { ...this.exportConfig.context } + ); + }); + } - if (this.workflows === undefined || isEmpty(this.workflows)) { - log.info(messageHandler.parse('WORKFLOW_NOT_FOUND'), this.exportConfig.context); - } else { - const workflowsFilePath = pResolve(this.webhooksFolderPath, this.workflowConfig.fileName); - log.debug(`Writing workflows to: ${workflowsFilePath}`, this.exportConfig.context); - fsUtil.writeFile(workflowsFilePath, this.workflows); - log.success( - messageHandler.parse('WORKFLOW_EXPORT_COMPLETE', Object.keys(this.workflows).length ), - this.exportConfig.context, - ); + async getWorkflowRoles(workflow: Record) { + log.debug(`Processing workflow roles for workflow: ${workflow.uid}`, this.exportConfig.context); + + for (const stage of workflow?.workflow_stages) { + log.debug(`Processing workflow stage: ${stage.name}`, this.exportConfig.context); + + for (let i = 0; i < stage?.SYS_ACL?.roles?.uids?.length; i++) { + const roleUid = stage.SYS_ACL.roles.uids[i]; + log.debug(`Fetching role data for role UID: ${roleUid}`, this.exportConfig.context); + const roleData = await this.getRoles(roleUid); + stage.SYS_ACL.roles.uids[i] = roleData; + } } } @@ -62,7 +69,7 @@ export default class ExportWorkFlows extends BaseClass { .workflow() .fetchAll(this.qs) .then(async (data: any) => { - const { items, count } = data; + const { count, items } = data; //NOTE - Handle the case where old workflow api is enabled in that case getting responses as objects. const workflowCount = count !== undefined ? count : items.length; log.debug(`Fetched ${items?.length || 0} workflows out of total ${workflowCount}`, this.exportConfig.context); @@ -106,37 +113,30 @@ export default class ExportWorkFlows extends BaseClass { log.debug(`Sanitization complete. Total workflows processed: ${Object.keys(this.workflows).length}`, this.exportConfig.context); } - async getWorkflowRoles(workflow: Record) { - log.debug(`Processing workflow roles for workflow: ${workflow.uid}`, this.exportConfig.context); - - for (const stage of workflow?.workflow_stages) { - log.debug(`Processing workflow stage: ${stage.name}`, this.exportConfig.context); - - for (let i = 0; i < stage?.SYS_ACL?.roles?.uids?.length; i++) { - const roleUid = stage.SYS_ACL.roles.uids[i]; - log.debug(`Fetching role data for role UID: ${roleUid}`, this.exportConfig.context); - const roleData = await this.getRoles(roleUid); - stage.SYS_ACL.roles.uids[i] = roleData; - } - } - } + async start(): Promise { + this.webhooksFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.workflowConfig.dirName, + ); + log.debug(`Workflows folder path: ${this.webhooksFolderPath}`, this.exportConfig.context); - async getRoles(roleUid: number): Promise { - log.debug(`Fetching role with UID: ${roleUid}`, this.exportConfig.context); + await fsUtil.makeDirectory(this.webhooksFolderPath); + log.debug('Created workflows directory', this.exportConfig.context); - return await this.stack - .role(roleUid) - .fetch({ include_rules: true, include_permissions: true }) - .then((data: any) => { - log.debug(`Successfully fetched role data for UID: ${roleUid}`, this.exportConfig.context); - return data; - }) - .catch((err: any) => { - log.debug(`Failed to fetch role data for UID: ${roleUid}`, this.exportConfig.context); - handleAndLogError( - err, - { ...this.exportConfig.context } - ); - }); + await this.getWorkflows(); + log.debug(`Retrieved ${Object.keys(this.workflows).length} workflows`, this.exportConfig.context); + + if (this.workflows === undefined || isEmpty(this.workflows)) { + log.info(messageHandler.parse('WORKFLOW_NOT_FOUND'), this.exportConfig.context); + } else { + const workflowsFilePath = pResolve(this.webhooksFolderPath, this.workflowConfig.fileName); + log.debug(`Writing workflows to: ${workflowsFilePath}`, this.exportConfig.context); + fsUtil.writeFile(workflowsFilePath, this.workflows); + log.success( + messageHandler.parse('WORKFLOW_EXPORT_COMPLETE', Object.keys(this.workflows).length ), + this.exportConfig.context, + ); + } } } diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index 12090760c..61aebe620 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -5,10 +5,23 @@ interface AnyProperty { } export default interface DefaultConfig { - contentVersion: number; - versioning: boolean; - host: string; + apis: { + assets: string; + content_types: string; + entries: string; + environments: string; + extension: string; + globalfields: string; + labels: string; + locales: string; + stacks: string; + userSession: string; + users: string; + webhooks: string; + }; cdn?: string; + contentVersion: number; + developerHubBaseUrl: string; developerHubUrls: any; // use below hosts for eu region // host:'https://eu-api.contentstack.com/v3', @@ -17,209 +30,203 @@ export default interface DefaultConfig { // use below hosts for gcp-na region // host: 'https://gcp-na-api.contentstack.com' // use below hosts for gcp-eu region + fetchConcurrency: number; + host: string; + languagesCode: string[]; + marketplaceAppEncryptionKey: string; // host: 'https://gcp-eu-api.contentstack.com' modules: { - types: Modules[]; - locales: { - dirName: string; - fileName: string; - requiredKeys: string[]; - dependencies?: Modules[]; - }; - customRoles: { - dirName: string; - fileName: string; - customRolesLocalesFileName: string; + assets: { + assetsMetaKeys: string[]; // Default keys ['uid', 'url', 'filename'] + // This is the total no. of asset objects fetched in each 'get assets' call + batchLimit: number; + // no of asset version files (of a single asset) that'll be downloaded parallel + chunkFileSize: number; // measured on Megabits (5mb) dependencies?: Modules[]; - }; - 'custom-roles': { dirName: string; + displayExecutionTime: boolean; + downloadLimit: number; + enableDownloadStatus: boolean; + fetchConcurrency: number; fileName: string; - customRolesLocalesFileName: string; - dependencies?: Modules[]; + host: string; + includeVersionedAssets: boolean; + invalidKeys: string[]; + securedAssets: boolean; }; - environments: { - dirName: string; - fileName: string; + attributes: { dependencies?: Modules[]; - }; - labels: { dirName: string; fileName: string; invalidKeys: string[]; - dependencies?: Modules[]; }; - webhooks: { - dirName: string; - fileName: string; + audiences: { dependencies?: Modules[]; - }; - releases: { dirName: string; fileName: string; - releasesList: string; invalidKeys: string[]; - dependencies?: Modules[]; }; - workflows: { + 'composable-studio': { + apiBaseUrl: string; + apiVersion: string; dirName: string; fileName: string; - invalidKeys: string[]; - dependencies?: Modules[]; }; - globalfields: { - dirName: string; - fileName: string; - validKeys: string[]; + content_types: { dependencies?: Modules[]; - }; - 'global-fields': { dirName: string; fileName: string; + // total no of content types fetched in each 'get content types' call + limit: number; validKeys: string[]; - dependencies?: Modules[]; }; - assets: { - dirName: string; - fileName: string; - // This is the total no. of asset objects fetched in each 'get assets' call - batchLimit: number; - host: string; - invalidKeys: string[]; - // no of asset version files (of a single asset) that'll be downloaded parallel - chunkFileSize: number; // measured on Megabits (5mb) - downloadLimit: number; - fetchConcurrency: number; - assetsMetaKeys: string[]; // Default keys ['uid', 'url', 'filename'] - securedAssets: boolean; - displayExecutionTime: boolean; - enableDownloadStatus: boolean; - includeVersionedAssets: boolean; + 'content-types': { dependencies?: Modules[]; - }; - content_types: { dirName: string; fileName: string; - validKeys: string[]; // total no of content types fetched in each 'get content types' call limit: number; - dependencies?: Modules[]; + validKeys: string[]; }; - 'content-types': { + 'custom-roles': { + customRolesLocalesFileName: string; + dependencies?: Modules[]; dirName: string; fileName: string; - validKeys: string[]; - // total no of content types fetched in each 'get content types' call - limit: number; + }; + customRoles: { + customRolesLocalesFileName: string; dependencies?: Modules[]; + dirName: string; + fileName: string; + }; + dependency: { + entries: string[]; }; entries: { + batchLimit: number; + dependencies?: Modules[]; dirName: string; + downloadLimit: number; + exportVersions: boolean; fileName: string; invalidKeys: string[]; - batchLimit: number; - downloadLimit: number; // total no of entries fetched in each content type in a single call limit: number; - dependencies?: Modules[]; - exportVersions: boolean; }; - personalize: { + environments: { + dependencies?: Modules[]; dirName: string; - baseURL: Record; - } & AnyProperty; - variantEntry: { + fileName: string; + }; + events: { + dependencies?: Modules[]; dirName: string; fileName: string; - chunkFileSize: number; - query: { - skip: number; - limit: number; - include_variant: boolean; - include_count: boolean; - include_publish_details: boolean; - } & AnyProperty; - } & AnyProperty; + invalidKeys: string[]; + }; extensions: { + dependencies?: Modules[]; dirName: string; fileName: string; + }; + 'global-fields': { dependencies?: Modules[]; + dirName: string; + fileName: string; + validKeys: string[]; }; - stack: { + globalfields: { + dependencies?: Modules[]; dirName: string; fileName: string; + validKeys: string[]; + }; + labels: { dependencies?: Modules[]; + dirName: string; + fileName: string; + invalidKeys: string[]; }; - dependency: { - entries: string[]; + locales: { + dependencies?: Modules[]; + dirName: string; + fileName: string; + requiredKeys: string[]; }; marketplace_apps: { + dependencies?: Modules[]; dirName: string; fileName: string; - dependencies?: Modules[]; }; 'marketplace-apps': { - dirName: string; - fileName: string; dependencies?: Modules[]; - }; - 'composable-studio': { dirName: string; fileName: string; - apiBaseUrl: string; - apiVersion: string; }; masterLocale: { dirName: string; fileName: string; requiredKeys: string[]; }; - taxonomies: { + personalize: { + baseURL: Record; + dirName: string; + } & AnyProperty; + 'publishing-rules': { + dependencies?: Modules[]; dirName: string; fileName: string; invalidKeys: string[]; - dependencies?: Modules[]; - limit: number; + limit?: number; }; - events: { + releases: { + dependencies?: Modules[]; dirName: string; fileName: string; invalidKeys: string[]; + releasesList: string; + }; + stack: { dependencies?: Modules[]; + dirName: string; + fileName: string; }; - audiences: { + taxonomies: { + dependencies?: Modules[]; dirName: string; fileName: string; invalidKeys: string[]; + limit: number; + }; + types: Modules[]; + variantEntry: { + chunkFileSize: number; + dirName: string; + fileName: string; + query: { + include_count: boolean; + include_publish_details: boolean; + include_variant: boolean; + limit: number; + skip: number; + } & AnyProperty; + } & AnyProperty; + webhooks: { dependencies?: Modules[]; + dirName: string; + fileName: string; }; - attributes: { + workflows: { + dependencies?: Modules[]; dirName: string; fileName: string; invalidKeys: string[]; - dependencies?: Modules[]; }; }; - languagesCode: string[]; - apis: { - userSession: string; - globalfields: string; - locales: string; - labels: string; - environments: string; - assets: string; - content_types: string; - entries: string; - users: string; - extension: string; - webhooks: string; - stacks: string; - }; - preserveStackVersion: boolean; + onlyTSModules: string[]; personalizationEnabled: boolean; - fetchConcurrency: number; + preserveStackVersion: boolean; + versioning: boolean; writeConcurrency: number; - developerHubBaseUrl: string; - marketplaceAppEncryptionKey: string; - onlyTSModules: string[]; } diff --git a/packages/contentstack-export/src/types/export-config.ts b/packages/contentstack-export/src/types/export-config.ts index 8b0e1b37b..7e0ce6892 100644 --- a/packages/contentstack-export/src/types/export-config.ts +++ b/packages/contentstack-export/src/types/export-config.ts @@ -2,45 +2,45 @@ import { Context, Modules, Region } from '.'; import DefaultConfig from './default-config'; export default interface ExportConfig extends DefaultConfig { - context: Context; - cliLogsPath: string; - exportDir: string; - data: string; - management_token?: string; + access_token?: string; apiKey: string; - forceStopMarketplaceAppsPrompt: boolean; auth_token?: string; - branchName?: string; + authenticationMethod?: string; branchAlias?: string; - securedAssets?: boolean; - contentTypes?: string[]; - branches?: branch[]; - branchEnabled?: boolean; branchDir?: string; - singleModuleExport?: boolean; - moduleName?: Modules; - master_locale: masterLocale; - query?: any; // Added query field + branchEnabled?: boolean; + branchName?: string; + branches?: branch[]; + cliLogsPath: string; + contentTypes?: string[]; + context: Context; + data: string; + exportDir: string; + forceStopMarketplaceAppsPrompt: boolean; headers?: { - api_key: string; + 'X-User-Agent': string; access_token?: string; + api_key: string; authtoken?: string; - 'X-User-Agent': string; organization_uid?: string; }; - access_token?: string; + management_token?: string; + master_locale: masterLocale; + moduleName?: Modules; org_uid?: string; - source_stack?: string; - sourceStackName?: string; + query?: any; // Added query field region: Region; - skipStackSettings?: boolean; + securedAssets?: boolean; + singleModuleExport?: boolean; skipDependencies?: boolean; - authenticationMethod?: string; + skipStackSettings?: boolean; + source_stack?: string; + sourceStackName?: string; } type branch = { - uid: string; source: string; + uid: string; }; type masterLocale = { diff --git a/packages/contentstack-export/src/types/index.ts b/packages/contentstack-export/src/types/index.ts index ec2118510..592aae960 100644 --- a/packages/contentstack-export/src/types/index.ts +++ b/packages/contentstack-export/src/types/index.ts @@ -1,4 +1,5 @@ import { ContentstackClient } from '@contentstack/cli-utilities'; + import ExportConfig from './export-config'; // eslint-disable-next-line @typescript-eslint/no-redeclare @@ -15,135 +16,147 @@ export interface PrintOptions { } export interface InquirePayload { - type: string; - name: string; - message: string; choices?: Array; - transformer?: Function; + message: string; + name: string; + transformer?: (value: string, answers: Record) => boolean | string; + type: string; } export interface User { - email: string; authtoken: string; + email: string; } export interface Region { - name: string; - cma: string; cda: string; + cma: string; + name: string; uiHost: string; } export type Modules = - | 'stack' | 'assets' - | 'locales' + | 'composable-studio' + | 'content-types' + | 'custom-roles' + | 'entries' | 'environments' | 'extensions' - | 'webhooks' | 'global-fields' - | 'entries' - | 'content-types' - | 'custom-roles' - | 'workflows' | 'labels' + | 'locales' | 'marketplace-apps' - | 'taxonomies' | 'personalize' - | 'composable-studio'; + | 'publishing-rules' + | 'stack' + | 'taxonomies' + | 'webhooks' + | 'workflows'; export type ModuleClassParams = { - stackAPIClient: ReturnType; exportConfig: ExportConfig; moduleName: Modules; + stackAPIClient: ReturnType; }; export interface ExternalConfig extends ExportConfig { + branchName: string; + data: string; + email?: string; + fetchConcurrency: number; master_locale: { - name: string; code: string; + name: string; }; - source_stack?: string; - data: string; - branchName: string; moduleName: Modules; - fetchConcurrency: number; - writeConcurrency: number; - securedAssets: boolean; - email?: string; password?: string; + securedAssets: boolean; + source_stack?: string; + writeConcurrency: number; } export interface ExtensionsConfig { + dependencies?: Modules[]; dirName: string; fileName: string; - dependencies?: Modules[]; limit?: number; } export interface MarketplaceAppsConfig { + dependencies?: Modules[]; dirName: string; fileName: string; - dependencies?: Modules[]; } export interface EnvironmentConfig { + dependencies?: Modules[]; dirName: string; fileName: string; - dependencies?: Modules[]; limit?: number; } export interface LabelConfig { + dependencies?: Modules[]; dirName: string; fileName: string; invalidKeys: string[]; - dependencies?: Modules[]; limit?: number; } export interface WebhookConfig { + dependencies?: Modules[]; dirName: string; fileName: string; - dependencies?: Modules[]; limit?: number; } export interface WorkflowConfig { + dependencies?: Modules[]; dirName: string; fileName: string; invalidKeys: string[]; - dependencies?: Modules[]; limit?: number; } -export interface CustomRoleConfig { +export interface PublishingRulesConfig { + dependencies?: Modules[]; dirName: string; fileName: string; + invalidKeys: string[]; + limit?: number; +} + +export interface CustomRoleConfig { customRolesLocalesFileName: string; dependencies?: Modules[]; + dirName: string; + fileName: string; } export interface StackConfig { + dependencies?: Modules[]; dirName: string; fileName: string; - dependencies?: Modules[]; limit?: number; } export interface ComposableStudioConfig { - dirName: string; - fileName: string; apiBaseUrl: string; apiVersion: string; + dirName: string; + fileName: string; } export interface ComposableStudioProject { - name: string; - description: string; canvasUrl: string; connectedStackApiKey: string; contentTypeUid: string; + createdAt: string; + createdBy: string; + deletedAt: boolean; + description: string; + name: string; organizationUid: string; settings: { configuration: { @@ -151,12 +164,9 @@ export interface ComposableStudioProject { locale: string; }; }; - createdBy: string; - updatedBy: string; - deletedAt: boolean; - createdAt: string; - updatedAt: string; uid: string; + updatedAt: string; + updatedBy: string; } export interface Context { module: string; diff --git a/packages/contentstack-export/src/types/marketplace-app.ts b/packages/contentstack-export/src/types/marketplace-app.ts index 684d9015a..ec7497a2d 100644 --- a/packages/contentstack-export/src/types/marketplace-app.ts +++ b/packages/contentstack-export/src/types/marketplace-app.ts @@ -1,35 +1,35 @@ type AppLocation = + | 'cs.cm.stack.asset_sidebar' | 'cs.cm.stack.config' - | 'cs.cm.stack.dashboard' - | 'cs.cm.stack.sidebar' | 'cs.cm.stack.custom_field' + | 'cs.cm.stack.dashboard' | 'cs.cm.stack.rte' - | 'cs.cm.stack.asset_sidebar' + | 'cs.cm.stack.sidebar' | 'cs.org.config'; interface ExtensionMeta { - uid?: string; - name?: string; + blur?: boolean; + data_type?: string; + default_width?: 'full' | 'half'; description?: string; + enabled?: boolean; + extension_uid?: string; + name?: string; path?: string; signed: boolean; - extension_uid?: string; - data_type?: string; - enabled?: boolean; + uid?: string; width?: number; - blur?: boolean; - default_width?: 'full' | 'half'; } interface Extension { - type: AppLocation; meta: ExtensionMeta[]; + type: AppLocation; } interface LocationConfiguration { - signed: boolean; base_url: string; locations: Extension[]; + signed: boolean; } interface AnyProperty { @@ -37,29 +37,29 @@ interface AnyProperty { } type Manifest = { - uid: string; - name: string; - icon?: string; - hosting?: any; - version?: number; description: string; - organization_uid: string; framework_version?: string; + hosting?: any; + icon?: string; + name: string; oauth?: any; - webhook?: any; + organization_uid: string; + target_type: 'organization' | 'stack'; ui_location: LocationConfiguration; - target_type: 'stack' | 'organization'; + uid: string; + version?: number; visibility: 'private' | 'public' | 'public_unlisted'; + webhook?: any; } & AnyProperty; type Installation = { - uid: string; - status: string; - manifest: Manifest; configuration: any; + manifest: Manifest; server_configuration: any; + status: string; target: { type: string; uid: string }; ui_location: LocationConfiguration; + uid: string; } & AnyProperty; export { Installation, Manifest }; diff --git a/packages/contentstack-export/src/utils/basic-login.ts b/packages/contentstack-export/src/utils/basic-login.ts index 6a9d9cabd..584b5debc 100644 --- a/packages/contentstack-export/src/utils/basic-login.ts +++ b/packages/contentstack-export/src/utils/basic-login.ts @@ -7,7 +7,8 @@ * MIT Licensed */ -import { log, managementSDKClient, authHandler } from '@contentstack/cli-utilities'; +import { authHandler, log, managementSDKClient } from '@contentstack/cli-utilities'; + import { ExternalConfig } from '../types'; const login = async (config: ExternalConfig): Promise => { @@ -16,16 +17,18 @@ const login = async (config: ExternalConfig): Promise => { const response = await client.login({ email: config.email, password: config.password }).catch(Promise.reject); if (response?.user?.authtoken) { config.headers = { - api_key: config.source_stack, + 'X-User-Agent': 'contentstack-export/v', access_token: config.access_token, + api_key: config.source_stack, authtoken: response.user.authtoken, - 'X-User-Agent': 'contentstack-export/v', }; await authHandler.setConfigData('basicAuth', response.user); log.success(`Contentstack account authenticated successfully!`, config.context); return config; } else { log.error(`Failed to log in!`, config.context); + // CLI: exit after unrecoverable auth failure (same behavior as before lint pass) + // eslint-disable-next-line n/no-process-exit -- intentional CLI termination process.exit(1); } } else if (!config.email && !config.password && config.source_stack && config.access_token) { @@ -38,9 +41,9 @@ const login = async (config: ExternalConfig): Promise => { config.context, ); config.headers = { - api_key: config.source_stack, - access_token: config.access_token, 'X-User-Agent': 'contentstack-export/v', + access_token: config.access_token, + api_key: config.source_stack, }; return config; } diff --git a/packages/contentstack-export/src/utils/common-helper.ts b/packages/contentstack-export/src/utils/common-helper.ts index 6c3de9a5b..9a4bd34fc 100644 --- a/packages/contentstack-export/src/utils/common-helper.ts +++ b/packages/contentstack-export/src/utils/common-helper.ts @@ -4,12 +4,12 @@ * MIT Licensed */ +import { getLogPath, isAuthenticated, sanitizePath } from '@contentstack/cli-utilities'; import * as path from 'path'; import promiseLimit from 'promise-limit'; -import { isAuthenticated, getLogPath, sanitizePath } from '@contentstack/cli-utilities'; +import { ExportConfig, ExternalConfig } from '../types'; import { fsUtil } from './file-helper'; -import { ExternalConfig, ExportConfig } from '../types'; export const validateConfig = function (config: ExternalConfig) { if (!config.host || !config.cdn) { diff --git a/packages/contentstack-export/src/utils/export-config-handler.ts b/packages/contentstack-export/src/utils/export-config-handler.ts index c67b6c12b..fbfcbfaad 100644 --- a/packages/contentstack-export/src/utils/export-config-handler.ts +++ b/packages/contentstack-export/src/utils/export-config-handler.ts @@ -1,12 +1,13 @@ +import { cliux, configHandler,isAuthenticated, log, sanitizePath } from '@contentstack/cli-utilities'; +import { filter, includes } from 'lodash'; import merge from 'merge'; import * as path from 'path'; -import { configHandler, isAuthenticated,cliux, sanitizePath, log } from '@contentstack/cli-utilities'; + import defaultConfig from '../config'; -import { readFile } from './file-helper'; -import { askExportDir, askAPIKey } from './interactive'; -import login from './basic-login'; -import { filter, includes } from 'lodash'; import { ExportConfig } from '../types'; +import login from './basic-login'; +import { readFile } from './file-helper'; +import { askAPIKey, askExportDir } from './interactive'; const setupConfig = async (exportCmdFlags: any): Promise => { let config = merge({}, defaultConfig); @@ -43,7 +44,7 @@ const setupConfig = async (exportCmdFlags: any): Promise => { if (managementTokenAlias) { log.debug('Using management token alias', { alias: managementTokenAlias }); - const { token, apiKey } = configHandler.get(`tokens.${managementTokenAlias}`) || {}; + const { apiKey, token } = configHandler.get(`tokens.${managementTokenAlias}`) || {}; config.management_token = token; config.apiKey = apiKey; authenticationMethod = 'Management Token'; diff --git a/packages/contentstack-export/src/utils/file-helper.ts b/packages/contentstack-export/src/utils/file-helper.ts index b96ee98c1..52a4541a7 100644 --- a/packages/contentstack-export/src/utils/file-helper.ts +++ b/packages/contentstack-export/src/utils/file-helper.ts @@ -1,8 +1,8 @@ +import { FsUtility, sanitizePath } from '@contentstack/cli-utilities'; +import bigJSON from 'big-json'; import * as fs from 'fs'; -import * as path from 'path'; import mkdirp from 'mkdirp'; -import bigJSON from 'big-json'; -import { FsUtility, sanitizePath } from '@contentstack/cli-utilities'; +import * as path from 'path'; export const readFileSync = function (filePath: string, parse: boolean): unknown { let data; @@ -81,7 +81,7 @@ export const writeLargeFile = function (filePath: string, data: any): Promise { resolve(''); @@ -92,9 +92,9 @@ export const writeLargeFile = function (filePath: string, data: any): Promise { return cliux.inquire({ - type: 'input', message: 'CLI_AUTH_LOGIN_ENTER_PASSWORD', name: 'password', transformer: (pswd: string) => { @@ -13,42 +12,43 @@ export const askPassword = async () => { } return pswdMasked; }, + type: 'input', }); }; export const askOTPChannel = async (): Promise => { return cliux.inquire({ - type: 'list', - name: 'otpChannel', - message: 'CLI_AUTH_LOGIN_ASK_CHANNEL_FOR_OTP', choices: [ { name: 'Authy App', value: 'authy' }, { name: 'SMS', value: 'sms' }, ], + message: 'CLI_AUTH_LOGIN_ASK_CHANNEL_FOR_OTP', + name: 'otpChannel', + type: 'list', }); }; export const askOTP = async (): Promise => { return cliux.inquire({ - type: 'input', message: 'CLI_AUTH_LOGIN_ENTER_SECURITY_CODE', name: 'tfaToken', + type: 'input', }); }; export const askUsername = async (): Promise => { return cliux.inquire({ - type: 'input', message: 'CLI_AUTH_LOGIN_ENTER_EMAIL_ADDRESS', name: 'username', + type: 'input', }); }; export const askExportDir = async (): Promise => { let result = await cliux.inquire({ - type: 'input', message: 'Enter the path for storing the content: (current folder)', name: 'dir', + type: 'input', validate: validatePath, }); if (!result) { @@ -61,8 +61,8 @@ export const askExportDir = async (): Promise => { export const askAPIKey = async (): Promise => { return await cliux.inquire({ - type: 'input', message: 'Enter the stack api key', name: 'apiKey', + type: 'input', }); }; diff --git a/packages/contentstack-export/src/utils/logger.ts b/packages/contentstack-export/src/utils/logger.ts index 07aa897b8..821be1d65 100644 --- a/packages/contentstack-export/src/utils/logger.ts +++ b/packages/contentstack-export/src/utils/logger.ts @@ -4,12 +4,12 @@ * MIT Licensed */ -import * as winston from 'winston'; -import * as path from 'path'; +import { redactObject, sanitizePath } from '@contentstack/cli-utilities'; import mkdirp from 'mkdirp'; +import * as path from 'path'; +import * as winston from 'winston'; + import { ExportConfig } from '../types'; -import { sanitizePath, redactObject } from '@contentstack/cli-utilities'; -const slice = Array.prototype.slice; const ansiRegexPattern = [ '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)', @@ -24,7 +24,7 @@ function returnString(args: unknown[]) { if (item && typeof item === 'object') { try { const redactedObject = redactObject(item); - if(redactedObject && typeof redactedObject === 'object') { + if (redactedObject && typeof redactedObject === 'object') { return JSON.stringify(redactedObject); } } catch (error) {} @@ -39,17 +39,17 @@ function returnString(args: unknown[]) { return returnStr; } const myCustomLevels = { - levels: { - warn: 1, - info: 2, - debug: 3, - }, colors: { + debug: 'green', + error: 'red', //colors aren't being used anywhere as of now, we're using chalk to add colors while logging info: 'blue', - debug: 'green', warn: 'yellow', - error: 'red', + }, + levels: { + debug: 3, + info: 2, + warn: 1, }, }; @@ -67,70 +67,66 @@ function init(_logPath: string) { successTransport = { filename: path.join(sanitizePath(logsDir), 'success.log'), + level: 'info', maxFiles: 20, maxsize: 1000000, tailable: true, - level: 'info', }; errorTransport = { filename: path.join(sanitizePath(logsDir), 'error.log'), + level: 'error', maxFiles: 20, maxsize: 1000000, tailable: true, - level: 'error', }; logger = winston.createLogger({ + levels: myCustomLevels.levels, transports: [ new winston.transports.File(successTransport), new winston.transports.Console({ format: winston.format.simple() }), ], - levels: myCustomLevels.levels, }); errorLogger = winston.createLogger({ + levels: { error: 0 }, transports: [ new winston.transports.File(errorTransport), new winston.transports.Console({ - level: 'error', format: winston.format.combine( winston.format.colorize({ all: true, colors: { error: 'red' } }), winston.format.simple(), ), + level: 'error', }), ], - levels: { error: 0 }, }); } return { - log: function (message: any) { - const args = slice.call(arguments); + debug: function (...args: unknown[]) { const logString = returnString(args); if (logString) { - logger.log('info', logString); + logger.log('debug', logString); } }, - warn: function () { - const args = slice.call(arguments); + error: function (...args: unknown[]) { const logString = returnString(args); if (logString) { - logger.log('warn', logString); + errorLogger.log('error', logString); } }, - error: function (message: any) { - const args = slice.call(arguments); + log: function (...args: unknown[]) { const logString = returnString(args); if (logString) { - errorLogger.log('error', logString); + logger.log('info', logString); } }, - debug: function () { - const args = slice.call(arguments); + warn: function (...args: unknown[]) { const logString = returnString(args); if (logString) { - logger.log('debug', logString); + logger.log('warn', logString); } }, }; diff --git a/packages/contentstack-export/src/utils/marketplace-app-helper.ts b/packages/contentstack-export/src/utils/marketplace-app-helper.ts index 970f35527..efbe1dc3f 100644 --- a/packages/contentstack-export/src/utils/marketplace-app-helper.ts +++ b/packages/contentstack-export/src/utils/marketplace-app-helper.ts @@ -1,4 +1,4 @@ -import { cliux, handleAndLogError, NodeCrypto, managementSDKClient, createDeveloperHubUrl } from '@contentstack/cli-utilities'; +import { NodeCrypto, cliux, createDeveloperHubUrl, handleAndLogError, managementSDKClient } from '@contentstack/cli-utilities'; import { ExportConfig } from '../types'; @@ -25,15 +25,15 @@ export async function createNodeCryptoInstance(config: ExportConfig): Promise { if (!url) return "Encryption key can't be empty."; return true; }, - message: 'Enter Marketplace app configurations encryption key', }); } diff --git a/packages/contentstack-export/src/utils/setup-branches.ts b/packages/contentstack-export/src/utils/setup-branches.ts index e5097800c..d9183901c 100644 --- a/packages/contentstack-export/src/utils/setup-branches.ts +++ b/packages/contentstack-export/src/utils/setup-branches.ts @@ -1,8 +1,8 @@ -import * as path from 'path'; import { sanitizePath } from '@contentstack/cli-utilities'; +import * as path from 'path'; import { ExportConfig } from '../types'; -import { writeFileSync, makeDirectory } from './file-helper'; +import { makeDirectory, writeFileSync } from './file-helper'; const setupBranches = async (config: ExportConfig, stackAPIClient: any) => { if (typeof config !== 'object') { @@ -15,6 +15,7 @@ const setupBranches = async (config: ExportConfig, stackAPIClient: any) => { const result = await stackAPIClient .branch(config.branchName) .fetch() + // eslint-disable-next-line @typescript-eslint/no-empty-function -- ignore branch fetch failure; handled below .catch((_err: Error) => {}); if (result && typeof result === 'object') { branches.push(result); @@ -27,6 +28,7 @@ const setupBranches = async (config: ExportConfig, stackAPIClient: any) => { .branch() .query() .find() + // eslint-disable-next-line @typescript-eslint/no-empty-function -- ignore list fetch failure; handled below .catch((_err: Error) => {}); if (result && result.items && Array.isArray(result.items) && result.items.length > 0) { branches = result.items; diff --git a/packages/contentstack-export/src/utils/setup-export-dir.ts b/packages/contentstack-export/src/utils/setup-export-dir.ts index 76de5b364..5d8332ab0 100644 --- a/packages/contentstack-export/src/utils/setup-export-dir.ts +++ b/packages/contentstack-export/src/utils/setup-export-dir.ts @@ -1,5 +1,5 @@ -import path from 'path'; import { sanitizePath } from '@contentstack/cli-utilities'; +import path from 'path'; import { ExportConfig } from '../types'; import { makeDirectory } from './file-helper'; diff --git a/packages/contentstack-export/test/unit/export/modules/assets.test.ts b/packages/contentstack-export/test/unit/export/modules/assets.test.ts index 56ef04ef7..408c9a63c 100644 --- a/packages/contentstack-export/test/unit/export/modules/assets.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/assets.test.ts @@ -113,6 +113,11 @@ describe('ExportAssets', () => { fileName: 'workflows.json', invalidKeys: [] }, + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: [], + }, globalfields: { dirName: 'global_fields', fileName: 'globalfields.json', diff --git a/packages/contentstack-export/test/unit/export/modules/base-class.test.ts b/packages/contentstack-export/test/unit/export/modules/base-class.test.ts index 0ffe4187f..4b62b5230 100644 --- a/packages/contentstack-export/test/unit/export/modules/base-class.test.ts +++ b/packages/contentstack-export/test/unit/export/modules/base-class.test.ts @@ -131,6 +131,11 @@ describe('BaseClass', () => { fileName: 'workflows.json', invalidKeys: [] }, + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: [], + }, globalfields: { dirName: 'global_fields', fileName: 'globalfields.json', diff --git a/packages/contentstack-export/test/unit/export/modules/publishing-rules.test.ts b/packages/contentstack-export/test/unit/export/modules/publishing-rules.test.ts new file mode 100644 index 000000000..e6ab1321b --- /dev/null +++ b/packages/contentstack-export/test/unit/export/modules/publishing-rules.test.ts @@ -0,0 +1,180 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { resolve as pResolve } from 'node:path'; +import { FsUtility } from '@contentstack/cli-utilities'; +import ExportPublishingRules from '../../../../src/export/modules/publishing-rules'; +import ExportConfig from '../../../../src/types/export-config'; + +describe('ExportPublishingRules', () => { + let exportPublishingRules: ExportPublishingRules; + let mockStackClient: any; + let mockExportConfig: ExportConfig; + + beforeEach(() => { + mockStackClient = { + workflow: sinon.stub().returns({ + publishRule: sinon.stub().returns({ + fetchAll: sinon.stub().resolves({ items: [], count: 0 }), + }), + }), + }; + + mockExportConfig = { + contentVersion: 1, + versioning: false, + host: 'https://api.contentstack.io', + developerHubUrls: {}, + apiKey: 'test-api-key', + exportDir: '/test/export', + data: '/test/data', + branchName: '', + context: { + command: 'cm:stacks:export', + module: 'publishing-rules', + userId: 'user-123', + email: 'test@example.com', + sessionId: 'session-123', + apiKey: 'test-api-key', + orgId: 'org-123', + authenticationMethod: 'Basic Auth', + }, + cliLogsPath: '/test/logs', + forceStopMarketplaceAppsPrompt: false, + master_locale: { code: 'en-us' }, + region: { + name: 'us', + cma: 'https://api.contentstack.io', + cda: 'https://cdn.contentstack.io', + uiHost: 'https://app.contentstack.com', + }, + skipStackSettings: false, + skipDependencies: false, + languagesCode: ['en'], + apis: {}, + preserveStackVersion: false, + personalizationEnabled: false, + fetchConcurrency: 5, + writeConcurrency: 5, + developerHubBaseUrl: '', + marketplaceAppEncryptionKey: '', + onlyTSModules: [], + modules: { + types: ['publishing-rules'], + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: ['stackHeaders', 'created_at'], + }, + }, + } as any; + + exportPublishingRules = new ExportPublishingRules({ + exportConfig: mockExportConfig, + stackAPIClient: mockStackClient, + moduleName: 'publishing-rules', + }); + + sinon.stub(FsUtility.prototype, 'writeFile').resolves(); + sinon.stub(FsUtility.prototype, 'makeDirectory').resolves(); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('Constructor', () => { + it('sets context.module to publishing-rules and reads module config', () => { + expect(exportPublishingRules).to.be.instanceOf(ExportPublishingRules); + expect(exportPublishingRules.exportConfig.context.module).to.equal('publishing-rules'); + expect((exportPublishingRules as any).publishingRulesConfig.fileName).to.equal('publishing-rules.json'); + expect((exportPublishingRules as any).publishingRulesConfig.dirName).to.equal('workflows'); + }); + }); + + describe('start()', () => { + it('resolves output path from data, branchName, and publishing-rules dirName', async () => { + const fetchAll = sinon.stub().resolves({ items: [], count: 0 }); + mockStackClient.workflow.returns({ + publishRule: sinon.stub().returns({ fetchAll }), + }); + + await exportPublishingRules.start(); + + const expectedFolder = pResolve(mockExportConfig.data, mockExportConfig.branchName || '', 'workflows'); + expect((FsUtility.prototype.makeDirectory as sinon.SinonStub).calledWith(expectedFolder)).to.be.true; + }); + + it('writes publishing-rules.json with rules omitting invalidKeys when API returns items', async () => { + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + const items = [ + { + uid: 'pr-1', + name: 'Rule 1', + stackHeaders: { h: 1 }, + created_at: '2020-01-01', + }, + ]; + mockStackClient.workflow.returns({ + publishRule: sinon.stub().returns({ + fetchAll: sinon.stub().resolves({ items, count: 1 }), + }), + }); + + await exportPublishingRules.start(); + + const expectedPath = pResolve( + mockExportConfig.data, + mockExportConfig.branchName || '', + 'workflows', + 'publishing-rules.json', + ); + expect(writeFileStub.calledOnce).to.be.true; + expect(writeFileStub.firstCall.args[0]).to.equal(expectedPath); + const written = writeFileStub.firstCall.args[1] as Record>; + expect(written['pr-1']).to.deep.equal({ uid: 'pr-1', name: 'Rule 1' }); + expect(written['pr-1'].stackHeaders).to.equal(undefined); + expect(written['pr-1'].created_at).to.equal(undefined); + }); + + it('does not write the rules file when no rules are returned', async () => { + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + mockStackClient.workflow.returns({ + publishRule: sinon.stub().returns({ + fetchAll: sinon.stub().resolves({ items: [], count: 0 }), + }), + }); + + await exportPublishingRules.start(); + + expect(writeFileStub.called).to.be.false; + }); + + it('requests the next page when count exceeds items length (pagination)', async () => { + const fetchAll = sinon.stub(); + fetchAll.onFirstCall().resolves({ + items: [ + { uid: 'a', name: 'A' }, + { uid: 'b', name: 'B' }, + ], + count: 3, + }); + fetchAll.onSecondCall().resolves({ + items: [{ uid: 'c', name: 'C' }], + count: 3, + }); + + mockStackClient.workflow.returns({ + publishRule: sinon.stub().returns({ fetchAll }), + }); + + await exportPublishingRules.start(); + + expect(fetchAll.callCount).to.equal(2); + expect(fetchAll.secondCall.args[0]).to.deep.include({ skip: 2, include_count: true }); + + const writeFileStub = FsUtility.prototype.writeFile as sinon.SinonStub; + const written = writeFileStub.firstCall.args[1] as Record>; + expect(Object.keys(written).sort((x, y) => x.localeCompare(y))).to.deep.equal(['a', 'b', 'c']); + }); + }); +}); diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 76b70e112..e8c9e2179 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -41,6 +41,7 @@ const config: DefaultConfig = { 'personalize', 'custom-roles', 'workflows', + 'publishing-rules', 'entries', 'variant-entries', 'labels', @@ -88,6 +89,11 @@ const config: DefaultConfig = { fileName: 'workflows.json', invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], }, + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: ['stackHeaders', 'urlPath', 'created_at', 'updated_at', 'created_by', 'updated_by'], + }, assets: { dirName: 'assets', assetBatchLimit: 1, @@ -455,5 +461,7 @@ const config: DefaultConfig = { globalModules: ['webhooks'], entriesPublish: true, }; +export const PUBLISHING_RULES_APPROVERS_SKIP_MSG = + 'Skipping import of publish rule approver(s) (roles/users); reconfigure approvers on the target stack.'; export default config; diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index a0eb77075..1981f1887 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -51,7 +51,8 @@ export type ApiModuleType = | 'delete-entries' | 'create-taxonomies' | 'create-terms' - | 'import-taxonomy'; + | 'import-taxonomy' + | 'create-publishing-rule'; export type ApiOptions = { uid?: string; @@ -374,6 +375,13 @@ export default abstract class BaseClass { .create({ workflow: apiData as WorkflowData }) .then(onSuccess) .catch(onReject); + case 'create-publishing-rule': + return this.stack + .workflow() + .publishRule() + .create({ publishing_rule: omit(apiData, ['uid']) as any }) + .then(onSuccess) + .catch(onReject); case 'create-custom-role': return this.stack .role() diff --git a/packages/contentstack-import/src/import/modules/publishing-rules.ts b/packages/contentstack-import/src/import/modules/publishing-rules.ts new file mode 100644 index 000000000..ca7734585 --- /dev/null +++ b/packages/contentstack-import/src/import/modules/publishing-rules.ts @@ -0,0 +1,275 @@ +import chalk from 'chalk'; +import values from 'lodash/values'; +import isEmpty from 'lodash/isEmpty'; +import { join } from 'node:path'; + +import BaseClass, { ApiOptions } from './base-class'; +import { PUBLISHING_RULES_APPROVERS_SKIP_MSG } from '../../config'; +import { fsUtil, fileHelper, parseErrorPayload, isDuplicatePublishingRuleError } from '../../utils'; +import { log, handleAndLogError } from '@contentstack/cli-utilities'; +import { ModuleClassParams, PublishingRulesConfig } from '../../types'; + +export default class ImportPublishingRules extends BaseClass { + private readonly mapperDirPath: string; + private readonly publishingRulesFolderPath: string; + private readonly publishingRulesUidMapperPath: string; + private readonly createdPublishingRulesPath: string; + private readonly failedPublishingRulesPath: string; + private readonly publishingRulesConfig: PublishingRulesConfig; + private publishingRules: Record; + private publishingRulesUidMapper: Record; + private readonly createdPublishingRules: Record[]; + private readonly failedPublishingRules: Record[]; + private envUidMapper: Record; + private workflowUidMapper: Record; + private readonly stageUidMapper: Record = {}; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); + this.importConfig.context.module = 'publishing-rules'; + this.publishingRulesConfig = importConfig.modules['publishing-rules']; + this.mapperDirPath = join(this.importConfig.backupDir, 'mapper', 'publishing-rules'); + this.publishingRulesFolderPath = join(this.importConfig.backupDir, this.publishingRulesConfig.dirName); + this.publishingRulesUidMapperPath = join(this.mapperDirPath, 'uid-mapping.json'); + this.createdPublishingRulesPath = join(this.mapperDirPath, 'success.json'); + this.failedPublishingRulesPath = join(this.mapperDirPath, 'fails.json'); + this.publishingRules = {}; + this.publishingRulesUidMapper = {}; + this.createdPublishingRules = []; + this.failedPublishingRules = []; + this.envUidMapper = {}; + this.workflowUidMapper = {}; + } + + private static collectOldStageUidToName( + exportedWorkflows: Record, + ): Record { + const map: Record = {}; + for (const workflow of Object.values(exportedWorkflows)) { + for (const stage of workflow.workflow_stages ?? []) { + if (stage.uid && stage.name) { + map[stage.uid] = stage.name; + } + } + } + return map; + } + + /** + * Returns `{ noSuccessMsg: true }` if any rule failed, so the import command skips the generic stack success line. + */ + async start(): Promise<{ noSuccessMsg: true } | void> { + const rulesFilePath = join(this.publishingRulesFolderPath, this.publishingRulesConfig.fileName); + + if (!fileHelper.fileExistsSync(rulesFilePath)) { + log.info(`No Publishing Rules found - '${rulesFilePath}'`, this.importConfig.context); + return; + } + + this.publishingRules = (fsUtil.readFile(rulesFilePath, true) as Record) ?? {}; + if (isEmpty(this.publishingRules)) { + log.info('No Publishing Rules found', this.importConfig.context); + return; + } + + await fsUtil.makeDirectory(this.mapperDirPath); + + this.publishingRulesUidMapper = this.readUidMappingFile(this.publishingRulesUidMapperPath); + this.envUidMapper = this.readMapper('environments'); + this.workflowUidMapper = this.readMapper('workflows'); + + await this.buildStageUidMapper(); + await this.importPublishingRules(); + + if (this.createdPublishingRules?.length) { + fsUtil.writeFile(this.createdPublishingRulesPath, this.createdPublishingRules); + } + if (this.failedPublishingRules?.length) { + fsUtil.writeFile(this.failedPublishingRulesPath, this.failedPublishingRules); + } + + const successCount = this.createdPublishingRules.length; + const failCount = this.failedPublishingRules.length; + + if (failCount > 0 && successCount === 0) { + log.error( + `Publishing rules import failed! ${failCount} rule(s) could not be imported. Check '${this.failedPublishingRulesPath}' for details.`, + this.importConfig.context, + ); + } else if (failCount > 0) { + log.warn( + `Publishing rules import completed with errors. Imported: ${successCount}, Failed: ${failCount}. Check '${this.failedPublishingRulesPath}' for details.`, + this.importConfig.context, + ); + } else { + log.success('Publishing rules have been imported successfully!', this.importConfig.context); + } + + if (failCount > 0) { + return { noSuccessMsg: true }; + } + } + + private readUidMappingFile(path: string): Record { + return fileHelper.fileExistsSync(path) ? (fsUtil.readFile(path, true) as Record) ?? {} : {}; + } + + private readMapper(moduleDir: string): Record { + const p = join(this.importConfig.backupDir, 'mapper', moduleDir, 'uid-mapping.json'); + return this.readUidMappingFile(p); + } + + private async importPublishingRules(): Promise { + const apiContent = values(this.publishingRules) as Record[]; + log.debug(`Importing ${apiContent.length} publishing rule(s)`, this.importConfig.context); + + const onSuccess = ({ response, apiData }: { response: { uid: string }; apiData: { uid: string } }) => { + const { uid } = apiData; + this.createdPublishingRules.push(response as unknown as Record); + this.publishingRulesUidMapper[uid] = response.uid; + log.success(`Publishing rule imported successfully (${uid} → ${response.uid})`, this.importConfig.context); + fsUtil.writeFile(this.publishingRulesUidMapperPath, this.publishingRulesUidMapper); + }; + + const onReject = ({ error, apiData }: { error: unknown; apiData: Record }) => { + const uid = apiData.uid as string; + const parsed = parseErrorPayload(error); + + if (isDuplicatePublishingRuleError(parsed, error)) { + log.info(`Publishing rule '${uid}' already exists`, this.importConfig.context); + return; + } + + this.failedPublishingRules.push(apiData); + handleAndLogError( + error as Error, + { ...this.importConfig.context, publishingRuleUid: uid }, + `Publishing rule '${uid}' failed to import`, + ); + }; + + await this.makeConcurrentCall( + { + apiContent, + processName: 'import publishing rules', + apiParams: { + serializeData: this.serializePublishingRules.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-publishing-rule', + includeParamOnCompletion: true, + }, + concurrencyLimit: this.importConfig.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + + private mergeFetchedWorkflowStages( + workflow: { workflow_stages?: { uid?: string; name?: string }[] }, + oldStageUidToName: Record, + ): void { + for (const newStage of workflow.workflow_stages ?? []) { + const oldUid = Object.keys(oldStageUidToName).find((u) => oldStageUidToName[u] === newStage.name); + if (oldUid && newStage.uid) { + this.stageUidMapper[oldUid] = newStage.uid; + } + } + } + + private async buildStageUidMapper(): Promise { + const wf = this.importConfig.modules.workflows as { dirName: string; fileName: string }; + const workflowsFilePath = join(this.importConfig.backupDir, wf.dirName, wf.fileName); + + if (!fileHelper.fileExistsSync(workflowsFilePath)) { + log.debug('No exported workflows file; stage UID mapping skipped', this.importConfig.context); + return; + } + + const exportedWorkflows = fsUtil.readFile(workflowsFilePath, true) as Record< + string, + { workflow_stages?: { uid?: string; name?: string }[] } + > | null; + if (!exportedWorkflows) return; + + const oldStageUidToName = ImportPublishingRules.collectOldStageUidToName(exportedWorkflows); + + for (const newWorkflowUid of Object.values(this.workflowUidMapper)) { + try { + const workflow = await this.stack.workflow(newWorkflowUid as string).fetch(); + this.mergeFetchedWorkflowStages( + workflow as { workflow_stages?: { uid?: string; name?: string }[] }, + oldStageUidToName, + ); + } catch (error: unknown) { + log.debug(`Stage mapping: could not fetch workflow '${newWorkflowUid}'`, this.importConfig.context); + handleAndLogError(error as Error, { ...this.importConfig.context }); + } + } + + log.debug(`Stage UID mapper: ${Object.keys(this.stageUidMapper).length} entr(y/ies)`, this.importConfig.context); + } + + private stripApprovers(rule: Record): void { + if (rule.approvers == null) return; + + const a = rule.approvers as { roles?: unknown[]; users?: unknown[] }; + const hadContent = (Array.isArray(a.roles) && a.roles.length > 0) || (Array.isArray(a.users) && a.users.length > 0); + if (hadContent) { + log.info(chalk.yellow(PUBLISHING_RULES_APPROVERS_SKIP_MSG), this.importConfig.context); + } + rule.approvers = { roles: [], users: [] }; + } + + private remapReference( + rule: Record, + field: 'workflow' | 'environment', + mapper: Record, + ): void { + const current = rule[field] as string | undefined; + if (!current) return; + const mapped = mapper[current] as string | undefined; + if (mapped) { + rule[field] = mapped; + log.debug(`${field} UID remapped`, this.importConfig.context); + } else { + log.debug(`No ${field} mapping for ${current}; leaving as-is`, this.importConfig.context); + } + } + + serializePublishingRules(apiOptions: ApiOptions): ApiOptions { + const rule = apiOptions.apiData as Record; + const ruleUid = rule.uid as string; + + if (ruleUid in this.publishingRulesUidMapper) { + log.info( + `Publishing rule '${ruleUid}' already exists. Skipping it to avoid duplicates!`, + this.importConfig.context, + ); + apiOptions.entity = undefined; + return apiOptions; + } + + const oldUid = ruleUid; + delete rule.uid; + + this.stripApprovers(rule); + this.remapReference(rule, 'workflow', this.workflowUidMapper); + this.remapReference(rule, 'environment', this.envUidMapper); + + if (rule.workflow_stage) { + const stage = rule.workflow_stage as string; + const mappedStage = this.stageUidMapper[stage]; + if (mappedStage) { + rule.workflow_stage = mappedStage; + log.debug('workflow_stage UID remapped', this.importConfig.context); + } else { + log.debug(`No workflow_stage mapping for ${stage}; leaving as-is`, this.importConfig.context); + } + } + + apiOptions.apiData = { ...rule, uid: oldUid }; + return apiOptions; + } +} diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index aa4867d29..e27968b9b 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -49,6 +49,11 @@ export default interface DefaultConfig { fileName: string; invalidKeys: string[]; }; + 'publishing-rules': { + dirName: string; + fileName: string; + invalidKeys: string[]; + }; assets: { dirName: string; assetBatchLimit: number; diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index ee9062465..8dac29ffa 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -46,6 +46,7 @@ export type Modules = | 'content-types' | 'custom-roles' | 'workflows' + | 'publishing-rules' | 'labels' | 'marketplace-apps' | 'taxonomies' @@ -94,6 +95,12 @@ export interface WorkflowConfig { invalidKeys: string[]; } +export interface PublishingRulesConfig { + dirName: string; + fileName: string; + invalidKeys: string[]; +} + export interface CustomRoleConfig { dirName: string; fileName: string; diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index fcf452a26..8232acd5d 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -33,3 +33,4 @@ export { export * from './common-helper'; export * from './log'; export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper'; +export { parseErrorPayload, isDuplicatePublishingRuleError } from './publishing-rules-helper'; diff --git a/packages/contentstack-import/src/utils/publishing-rules-helper.ts b/packages/contentstack-import/src/utils/publishing-rules-helper.ts new file mode 100644 index 000000000..b7f3b44ec --- /dev/null +++ b/packages/contentstack-import/src/utils/publishing-rules-helper.ts @@ -0,0 +1,32 @@ +/** + * Helpers for publishing rules import (API error shape, duplicate detection). + */ + +export function parseErrorPayload(error: unknown): { + errors?: Record; + error_message?: string; +} | null { + if (!error || typeof error !== 'object') return null; + const e = error as { message?: string; errors?: Record }; + if (e.errors) return e; + if (e.message && typeof e.message === 'string') { + try { + return JSON.parse(e.message) as { errors?: Record; error_message?: string }; + } catch { + return null; + } + } + return null; +} + +export function isDuplicatePublishingRuleError( + parsed: { errors?: Record; error_message?: string } | null, + raw: unknown, +): boolean { + const errors = parsed?.errors ?? (raw as { errors?: Record })?.errors; + if (errors?.name || errors?.['publishing_rule.name'] || errors?.['publish_rule.name']) { + return true; + } + const msg = parsed?.error_message; + return typeof msg === 'string' && /already exists|duplicate/i.test(msg); +} diff --git a/packages/contentstack-import/test/unit/import/modules/locales.test.ts b/packages/contentstack-import/test/unit/import/modules/locales.test.ts index fc3631e81..1f5da35cf 100644 --- a/packages/contentstack-import/test/unit/import/modules/locales.test.ts +++ b/packages/contentstack-import/test/unit/import/modules/locales.test.ts @@ -51,6 +51,11 @@ describe('ImportLocales', () => { webhooks: { dirName: 'webhooks', fileName: 'webhooks.json' }, releases: { dirName: 'releases', fileName: 'releases.json', invalidKeys: ['uid'] }, workflows: { dirName: 'workflows', fileName: 'workflows.json', invalidKeys: ['uid'] }, + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: ['uid'], + }, assets: { dirName: 'assets', assetBatchLimit: 10, diff --git a/packages/contentstack-import/test/unit/import/modules/publishing-rules.test.ts b/packages/contentstack-import/test/unit/import/modules/publishing-rules.test.ts new file mode 100644 index 000000000..26aa3f4a7 --- /dev/null +++ b/packages/contentstack-import/test/unit/import/modules/publishing-rules.test.ts @@ -0,0 +1,333 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { join } from 'node:path'; +import ImportPublishingRules from '../../../../src/import/modules/publishing-rules'; +import { ImportConfig } from '../../../../src/types'; +describe('ImportPublishingRules', () => { + const BACKUP = '/test/backup'; + const rulesFile = join(BACKUP, 'workflows', 'publishing-rules.json'); + const workflowsExportFile = join(BACKUP, 'workflows', 'workflows.json'); + const workflowMapperFile = join(BACKUP, 'mapper', 'workflows', 'uid-mapping.json'); + const envMapperFile = join(BACKUP, 'mapper', 'environments', 'uid-mapping.json'); + const publishingMapperFile = join(BACKUP, 'mapper', 'publishing-rules', 'uid-mapping.json'); + + let importPublishingRules: ImportPublishingRules; + let mockStackClient: any; + let mockImportConfig: ImportConfig; + let fsUtilStub: any; + let fileHelperStub: any; + let makeConcurrentCallStub: sinon.SinonStub; + let logStub: { info: sinon.SinonStub; debug: sinon.SinonStub; success: sinon.SinonStub; error: sinon.SinonStub; warn: sinon.SinonStub }; + beforeEach(() => { + fsUtilStub = { + readFile: sinon.stub(), + writeFile: sinon.stub(), + makeDirectory: sinon.stub().resolves(), + }; + + fileHelperStub = { + fileExistsSync: sinon.stub(), + }; + + sinon.replace(require('../../../../src/utils'), 'fileHelper', fileHelperStub); + sinon.replaceGetter(require('../../../../src/utils'), 'fsUtil', () => fsUtilStub); + + const fetchWorkflowStub = sinon.stub().resolves({ + workflow_stages: [{ uid: 'stage-new', name: 'Review' }], + }); + mockStackClient = { + workflow: sinon.stub().returns({ + fetch: fetchWorkflowStub, + }), + }; + + mockImportConfig = { + apiKey: 'test', + backupDir: BACKUP, + data: '/test/content', + contentVersion: 1, + region: 'us', + fetchConcurrency: 2, + context: { + command: 'cm:stacks:import', + module: 'publishing-rules', + userId: 'user-123', + email: 'test@example.com', + sessionId: 'session-123', + apiKey: 'test', + orgId: 'org-123', + authenticationMethod: 'Basic Auth', + }, + modules: { + workflows: { + dirName: 'workflows', + fileName: 'workflows.json', + invalidKeys: ['uid'], + }, + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: ['uid'], + }, + }, + } as any; + + importPublishingRules = new ImportPublishingRules({ + importConfig: mockImportConfig as any, + stackAPIClient: mockStackClient, + moduleName: 'publishing-rules', + }); + + makeConcurrentCallStub = sinon.stub(importPublishingRules as any, 'makeConcurrentCall').resolves(); + + const cliUtilities = require('@contentstack/cli-utilities'); + logStub = { + info: sinon.stub(), + debug: sinon.stub(), + success: sinon.stub(), + error: sinon.stub(), + warn: sinon.stub(), + }; + sinon.stub(cliUtilities, 'log').value(logStub); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe('Constructor', () => { + it('sets context.module to publishing-rules and derives exact paths from backupDir and config', () => { + expect(mockImportConfig.context.module).to.equal('publishing-rules'); + expect(importPublishingRules['mapperDirPath']).to.equal(join(BACKUP, 'mapper', 'publishing-rules')); + expect(importPublishingRules['publishingRulesFolderPath']).to.equal(join(BACKUP, 'workflows')); + expect(importPublishingRules['publishingRulesUidMapperPath']).to.equal(publishingMapperFile); + expect(importPublishingRules['createdPublishingRulesPath']).to.equal( + join(BACKUP, 'mapper', 'publishing-rules', 'success.json'), + ); + expect(importPublishingRules['failedPublishingRulesPath']).to.equal( + join(BACKUP, 'mapper', 'publishing-rules', 'fails.json'), + ); + }); + + it('initializes empty rules, mappers, and result arrays', () => { + expect(importPublishingRules['publishingRules']).to.deep.equal({}); + expect(importPublishingRules['publishingRulesUidMapper']).to.deep.equal({}); + expect(importPublishingRules['createdPublishingRules']).to.deep.equal([]); + expect(importPublishingRules['failedPublishingRules']).to.deep.equal([]); + expect(importPublishingRules['envUidMapper']).to.deep.equal({}); + expect(importPublishingRules['workflowUidMapper']).to.deep.equal({}); + expect(importPublishingRules['stageUidMapper']).to.deep.equal({}); + }); + }); + + describe('start()', () => { + it('returns undefined and logs missing file path when rules file does not exist', async () => { + fileHelperStub.fileExistsSync.withArgs(rulesFile).returns(false); + + const result = await importPublishingRules.start(); + + expect(result).to.equal(undefined); + expect(makeConcurrentCallStub.called).to.be.false; + expect(logStub.info.firstCall.args[0]).to.include(rulesFile); + }); + + it('returns undefined when rules file exists but payload is empty; arrays stay empty', async () => { + fileHelperStub.fileExistsSync.withArgs(rulesFile).returns(true); + fsUtilStub.readFile.withArgs(rulesFile, true).returns({}); + + const result = await importPublishingRules.start(); + + expect(result).to.equal(undefined); + expect(makeConcurrentCallStub.called).to.be.false; + expect(importPublishingRules['createdPublishingRules']).to.deep.equal([]); + expect(importPublishingRules['failedPublishingRules']).to.deep.equal([]); + }); + + it('passes one apiContent item per rule and binds serializeData to serializePublishingRules', async () => { + const rules = { + r1: { uid: 'r1', name: 'Rule 1' }, + r2: { uid: 'r2', name: 'Rule 2' }, + }; + fileHelperStub.fileExistsSync.callsFake((p: string) => { + if (p === rulesFile) return true; + if (p === workflowsExportFile || p === workflowMapperFile || p === envMapperFile || p === publishingMapperFile) { + return false; + } + return false; + }); + fsUtilStub.readFile.callsFake((p: string) => { + if (p === rulesFile) return rules; + return {}; + }); + + await importPublishingRules.start(); + + expect(makeConcurrentCallStub.calledOnce).to.be.true; + const callArgs = makeConcurrentCallStub.firstCall.args[0]; + expect(callArgs.apiContent).to.have.length(2); + expect(callArgs.processName).to.equal('import publishing rules'); + expect(callArgs.apiParams.entity).to.equal('create-publishing-rule'); + const serialized = callArgs.apiParams.serializeData({ + apiData: { uid: 'r1', name: 'Rule 1' }, + entity: 'create-publishing-rule', + }); + expect(serialized.apiData).to.deep.include({ name: 'Rule 1', uid: 'r1' }); + expect(serialized.entity).to.equal('create-publishing-rule'); + }); + + it('builds stageUidMapper from exported workflows and fetched target workflow stages by name', async () => { + fileHelperStub.fileExistsSync.callsFake((p: string) => { + if (p === rulesFile) return true; + if (p === workflowsExportFile) return true; + if (p === workflowMapperFile) return true; + if (p === envMapperFile || p === publishingMapperFile) return false; + return false; + }); + fsUtilStub.readFile.callsFake((p: string) => { + if (p === rulesFile) return { r1: { uid: 'r1', name: 'R' } }; + if (p === workflowsExportFile) { + return { + expWf: { workflow_stages: [{ uid: 'stage-old', name: 'Review' }] }, + }; + } + if (p === workflowMapperFile) return { oldWf: 'newWf' }; + return {}; + }); + + await importPublishingRules.start(); + + expect(importPublishingRules['stageUidMapper']).to.deep.equal({ 'stage-old': 'stage-new' }); + expect(mockStackClient.workflow.calledWith('newWf')).to.be.true; + }); + + it('returns { noSuccessMsg: true } when a rule fails to import (non-duplicate error)', async () => { + fileHelperStub.fileExistsSync.callsFake((p: string) => p === rulesFile); + fsUtilStub.readFile.callsFake((p: string) => (p === rulesFile ? { r1: { uid: 'r1', name: 'R' } } : {})); + + makeConcurrentCallStub.callsFake(async (env: any) => { + const { apiParams, apiContent } = env; + for (const element of apiContent) { + apiParams.apiData = element; + let opts = { ...apiParams, apiData: { ...element } }; + opts = apiParams.serializeData(opts); + if (opts.entity) { + await apiParams.reject({ error: new Error('network'), apiData: opts.apiData }); + } + } + }); + + const result = await importPublishingRules.start(); + + expect(result).to.deep.equal({ noSuccessMsg: true }); + expect(importPublishingRules['failedPublishingRules']).to.have.length(1); + expect(importPublishingRules['failedPublishingRules'][0].uid).to.equal('r1'); + expect(String(logStub.error.firstCall?.args[0] ?? '')).to.include('could not be imported'); + }); + + it('returns undefined when import succeeds with no failures', async () => { + fileHelperStub.fileExistsSync.callsFake((p: string) => p === rulesFile); + fsUtilStub.readFile.callsFake((p: string) => (p === rulesFile ? { r1: { uid: 'r1', name: 'R' } } : {})); + + makeConcurrentCallStub.callsFake(async (env: any) => { + const { apiParams, apiContent } = env; + for (const element of apiContent) { + apiParams.apiData = element; + let opts = { ...apiParams, apiData: { ...element } }; + opts = apiParams.serializeData(opts); + if (opts.entity) { + await apiParams.resolve({ response: { uid: 'new-r1' }, apiData: opts.apiData }); + } + } + }); + + const result = await importPublishingRules.start(); + + expect(result).to.equal(undefined); + expect(importPublishingRules['failedPublishingRules']).to.deep.equal([]); + expect(importPublishingRules['publishingRulesUidMapper']).to.deep.equal({ r1: 'new-r1' }); + expect(logStub.success.calledWith('Publishing rules have been imported successfully!', mockImportConfig.context)).to.be + .true; + }); + }); + + describe('serializePublishingRules', () => { + it('clears entity when rule uid already in mapper; leaves apiData.uid unchanged', () => { + importPublishingRules['publishingRulesUidMapper'] = { 'rule-1': 'mapped-1' }; + + const apiOptions: any = { + apiData: { uid: 'rule-1', name: 'N' }, + entity: 'create-publishing-rule', + }; + + const out = importPublishingRules.serializePublishingRules(apiOptions); + + expect(out.entity).to.equal(undefined); + expect(out.apiData).to.deep.equal({ uid: 'rule-1', name: 'N' }); + expect(String(logStub.info.firstCall?.args[0] ?? '')).to.match(/already exists\. Skipping/); + }); + + it('remaps workflow, environment, workflow_stage and strips approvers; apiData carries uid for completion handler', () => { + const pr = importPublishingRules as any; + pr.workflowUidMapper = { wfOld: 'wfNew' }; + pr.envUidMapper = { envOld: 'envNew' }; + Object.keys(pr.stageUidMapper).forEach((k) => delete pr.stageUidMapper[k]); + pr.stageUidMapper.stOld = 'stNew'; + + const apiOptions: any = { + apiData: { + uid: 'pr-1', + name: 'PR', + workflow: 'wfOld', + environment: 'envOld', + workflow_stage: 'stOld', + approvers: { roles: ['r1'], users: ['u1'] }, + }, + entity: 'create-publishing-rule', + }; + + const out = importPublishingRules.serializePublishingRules(apiOptions); + + expect(out.entity).to.equal('create-publishing-rule'); + expect(out.apiData).to.deep.equal({ + uid: 'pr-1', + name: 'PR', + workflow: 'wfNew', + environment: 'envNew', + workflow_stage: 'stNew', + approvers: { roles: [], users: [] }, + }); + const infoArgs = logStub.info.getCalls().map((c) => c.args[0]); + expect(infoArgs.some((msg) => String(msg).includes('Skipping import of publish rule approver'))).to.be.true; + }); + }); + + describe('importPublishingRules callbacks', () => { + beforeEach(() => { + importPublishingRules['publishingRules'] = { r1: { uid: 'r1', name: 'R' } }; + }); + + it('onSuccess updates mapper and persists uid-mapping.json with expected payload', async () => { + await (importPublishingRules as any).importPublishingRules(); + + const onSuccess = makeConcurrentCallStub.firstCall.args[0].apiParams.resolve; + await onSuccess({ response: { uid: 'created-uid', name: 'R' }, apiData: { uid: 'r1', name: 'R' } }); + + expect(importPublishingRules['createdPublishingRules']).to.deep.equal([{ uid: 'created-uid', name: 'R' }]); + expect(importPublishingRules['publishingRulesUidMapper']).to.deep.equal({ r1: 'created-uid' }); + expect(fsUtilStub.writeFile.calledWith(publishingMapperFile, { r1: 'created-uid' })).to.be.true; + }); + + it('onReject for duplicate error does not append to failedPublishingRules', async () => { + await (importPublishingRules as any).importPublishingRules(); + + const onReject = makeConcurrentCallStub.firstCall.args[0].apiParams.reject; + await onReject({ + error: { errors: { name: 'taken' } }, + apiData: { uid: 'r1', name: 'R' }, + }); + + expect(importPublishingRules['failedPublishingRules']).to.deep.equal([]); + expect(logStub.info.calledWith(`Publishing rule 'r1' already exists`, mockImportConfig.context)).to.be.true; + }); + }); +}); diff --git a/packages/contentstack-import/test/unit/utils/extension-helper.test.ts b/packages/contentstack-import/test/unit/utils/extension-helper.test.ts index c9b567218..fa537a84e 100644 --- a/packages/contentstack-import/test/unit/utils/extension-helper.test.ts +++ b/packages/contentstack-import/test/unit/utils/extension-helper.test.ts @@ -54,6 +54,11 @@ describe('Extension Helper', () => { webhooks: { dirName: 'webhooks', fileName: 'webhooks.json' }, releases: { dirName: 'releases', fileName: 'releases.json', invalidKeys: ['uid'] }, workflows: { dirName: 'workflows', fileName: 'workflows.json', invalidKeys: ['uid'] }, + 'publishing-rules': { + dirName: 'workflows', + fileName: 'publishing-rules.json', + invalidKeys: ['uid'], + }, assets: { dirName: 'assets', assetBatchLimit: 10, diff --git a/packages/contentstack-import/test/unit/utils/publishing-rules-helper.test.ts b/packages/contentstack-import/test/unit/utils/publishing-rules-helper.test.ts new file mode 100644 index 000000000..1453b68eb --- /dev/null +++ b/packages/contentstack-import/test/unit/utils/publishing-rules-helper.test.ts @@ -0,0 +1,57 @@ +import { expect } from 'chai'; +import { parseErrorPayload, isDuplicatePublishingRuleError } from '../../../src/utils/publishing-rules-helper'; + +describe('publishing-rules-helper', () => { + describe('parseErrorPayload', () => { + it('returns the object when error has errors', () => { + const err = { errors: { name: 'taken' } }; + expect(parseErrorPayload(err)).to.deep.equal({ errors: { name: 'taken' } }); + }); + + it('parses JSON from message string', () => { + const err = { message: JSON.stringify({ error_message: 'already exists' }) }; + expect(parseErrorPayload(err)).to.deep.equal({ error_message: 'already exists' }); + }); + + it('returns null for invalid JSON in message', () => { + expect(parseErrorPayload({ message: 'not-json{' })).to.equal(null); + }); + + it('returns null for non-object', () => { + expect(parseErrorPayload(null)).to.equal(null); + expect(parseErrorPayload('x')).to.equal(null); + }); + + it('returns null when object has no errors or parseable message', () => { + expect(parseErrorPayload({ foo: 1 })).to.equal(null); + }); + }); + + describe('isDuplicatePublishingRuleError', () => { + it('returns true when errors.name is set', () => { + expect(isDuplicatePublishingRuleError({ errors: { name: 'x' } }, {})).to.equal(true); + }); + + it('returns true when errors.publishing_rule.name is set', () => { + expect(isDuplicatePublishingRuleError({ errors: { 'publishing_rule.name': 'x' } }, {})).to.equal(true); + }); + + it('returns true when errors.publish_rule.name is set', () => { + expect(isDuplicatePublishingRuleError({ errors: { 'publish_rule.name': 'x' } }, {})).to.equal(true); + }); + + it('returns true when error_message matches duplicate wording', () => { + expect(isDuplicatePublishingRuleError({ error_message: 'Rule already exists' }, {})).to.equal(true); + }); + + it('reads errors from raw when parsed is null', () => { + const raw = { errors: { name: 'dup' } }; + expect(isDuplicatePublishingRuleError(null, raw)).to.equal(true); + }); + + it('returns false when no duplicate signals', () => { + expect(isDuplicatePublishingRuleError({ errors: { other: 'x' } }, {})).to.equal(false); + expect(isDuplicatePublishingRuleError({ error_message: 'timeout' }, {})).to.equal(false); + }); + }); +}); diff --git a/packages/contentstack-variants/src/types/export-config.ts b/packages/contentstack-variants/src/types/export-config.ts index f9336bc96..8b782c2b0 100644 --- a/packages/contentstack-variants/src/types/export-config.ts +++ b/packages/contentstack-variants/src/types/export-config.ts @@ -17,6 +17,7 @@ export type Modules = | 'content-types' | 'custom-roles' | 'workflows' + | 'publishing-rules' | 'labels' | 'marketplace-apps' | 'taxonomies' @@ -88,6 +89,13 @@ export interface DefaultConfig { invalidKeys: string[]; dependencies?: Modules[]; }; + 'publishing-rules': { + dirName: string; + fileName: string; + invalidKeys: string[]; + dependencies?: Modules[]; + limit?: number; + }; globalfields: { dirName: string; fileName: string;