#!/bin/sh set -eu usage() { cat <<'EOF' Usage: ./script/import-collection-schema.sh \ --env \ --input-dir \ [--apply --confirm-env ] Options: --env Required target NocoBase CLI environment name. --input-dir Required directory produced by export-collection-schema.sh. --apply Apply the validated plan. Default: plan only, no writes. --confirm-env Required with --apply and must exactly equal --env. -h, --help Show this help. Behavior: - Accepts only portable schemas for the main data source. - Validates manifest.json and every referenced schema before any write. - Applies collection shells and non-relation fields first, then relations. - Reads every imported collection back and verifies the declared schema. - Creates missing Collection categories and reconciles category memberships. - Treats the exported pseudo-category "未分类" as no category membership. - Does not delete extra fields already present in the target environment. Plan only: ./script/import-collection-schema.sh \ --env mdm-uat \ --input-dir ./exports/reference-schema Apply: ./script/import-collection-schema.sh \ --env mdm-uat \ --input-dir ./exports/reference-schema \ --apply \ --confirm-env mdm-uat EOF } ENV_NAME= INPUT_DIR= APPLY=0 CONFIRM_ENV= while [ "$#" -gt 0 ]; do case "$1" in --env) ENV_NAME=${2:?--env requires a value} shift 2 ;; --input-dir) INPUT_DIR=${2:?--input-dir requires a value} shift 2 ;; --apply) APPLY=1 shift ;; --confirm-env) CONFIRM_ENV=${2:?--confirm-env requires a value} shift 2 ;; -h|--help) usage exit 0 ;; *) echo "Unknown argument: $1" >&2 usage >&2 exit 2 ;; esac done [ -n "$ENV_NAME" ] || { echo "--env is required" >&2; exit 2; } [ -n "$INPUT_DIR" ] || { echo "--input-dir is required" >&2; exit 2; } [ -d "$INPUT_DIR" ] || { echo "Input directory does not exist: $INPUT_DIR" >&2; exit 1; } command -v nb >/dev/null 2>&1 || { echo "nb CLI is required" >&2; exit 2; } command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 2; } if [ "$APPLY" -eq 0 ] && [ -n "$CONFIRM_ENV" ]; then echo "--confirm-env is only valid together with --apply" >&2 exit 2 fi if [ "$APPLY" -eq 1 ] && [ "$CONFIRM_ENV" != "$ENV_NAME" ]; then echo "--apply requires --confirm-env to exactly equal --env ($ENV_NAME)" >&2 exit 2 fi INPUT_DIR=$(CDPATH= cd -- "$INPUT_DIR" && pwd) MANIFEST_FILE="$INPUT_DIR/manifest.json" [ -f "$MANIFEST_FILE" ] && [ ! -L "$MANIFEST_FILE" ] || { echo "Regular manifest.json not found in: $INPUT_DIR" >&2 exit 1 } if ! jq -e ' type == "object" and .profile == "portable" and .dataSource == "main" and (.files | type == "array" and length > 0) and all(.files[]; .dataSource == "main" and (.category | type == "string" and length > 0) and (.collection | type == "string" and length > 0) and (.file | type == "string" and length > 0) and (.file | startswith("/") | not) and (.file | contains("\\") | not) and (.file | split("/") | all(. != "" and . != "." and . != "..")) ) and (([.files[].file] | unique | length) == (.files | length)) and ((has("exportedFiles") | not) or .exportedFiles == (.files | length)) ' "$MANIFEST_FILE" >/dev/null; then echo "Invalid manifest.json: portable main-data-source export required" >&2 exit 1 fi WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/import-collection-schema.XXXXXX") ROWS_FILE="$WORK_DIR/schema-rows.jsonl" PLAN_FILE="$WORK_DIR/plan.json" VERIFY_FILE="$WORK_DIR/verification-errors.jsonl" : > "$ROWS_FILE" : > "$VERIFY_FILE" trap 'rm -rf "$WORK_DIR"' EXIT HUP INT TERM FILE_COUNT=$(jq '.files | length' "$MANIFEST_FILE") INDEX=0 while [ "$INDEX" -lt "$FILE_COUNT" ]; do ITEM=$(jq -c ".files[$INDEX]" "$MANIFEST_FILE") RELATIVE_FILE=$(printf '%s' "$ITEM" | jq -r '.file') COLLECTION_NAME=$(printf '%s' "$ITEM" | jq -r '.collection') CATEGORY_NAME=$(printf '%s' "$ITEM" | jq -r '.category') SCHEMA_FILE="$INPUT_DIR/$RELATIVE_FILE" [ -f "$SCHEMA_FILE" ] && [ ! -L "$SCHEMA_FILE" ] || { echo "Schema file is missing or is not a regular file: $RELATIVE_FILE" >&2 exit 1 } if ! jq -e --arg collection "$COLLECTION_NAME" ' type == "object" and .name == $collection and (.title | type == "string" and length > 0) and (.template | type == "string" and length > 0) and (.fields | type == "array") and all(.fields[]; type == "object" and (.name | type == "string" and length > 0) and (.interface | type == "string" and length > 0) ) and (([.fields[].name] | unique | length) == (.fields | length)) ' "$SCHEMA_FILE" >/dev/null; then echo "Invalid Collection schema: $RELATIVE_FILE" >&2 exit 1 fi jq -c \ --arg collection "$COLLECTION_NAME" \ --arg category "$CATEGORY_NAME" \ --arg file "$RELATIVE_FILE" \ '{collection:$collection, category:$category, file:$file, schema:.}' \ "$SCHEMA_FILE" >> "$ROWS_FILE" INDEX=$((INDEX + 1)) done if ! jq -se ' sort_by(.collection) | group_by(.collection) | all(.[]; ([.[].schema] | unique | length) == 1) ' "$ROWS_FILE" >/dev/null; then echo "One Collection is represented by conflicting schema files" >&2 exit 1 fi jq -s ' sort_by(.collection) | group_by(.collection) | map({ collection: .[0].collection, categories: ([.[].category] | unique | sort), files: ([.[].file] | unique | sort), schema: .[0].schema }) ' "$ROWS_FILE" > "$PLAN_FILE" COLLECTION_COUNT=$(jq 'length' "$PLAN_FILE") if ! jq -e --argjson count "$COLLECTION_COUNT" ' (has("exportedCollections") | not) or .exportedCollections == $count ' "$MANIFEST_FILE" >/dev/null; then echo "manifest.json exportedCollections does not match the schemas" >&2 exit 1 fi RELATION_INTERFACES='["m2o","o2m","m2m","o2o"]' SCALAR_FIELD_COUNT=$(jq --argjson relations "$RELATION_INTERFACES" ' [.[] | .schema.fields[] | select((.interface as $i | $relations | index($i)) == null)] | length ' "$PLAN_FILE") RELATION_FIELD_COUNT=$(jq --argjson relations "$RELATION_INTERFACES" ' [.[] | .schema.fields[] | select(.interface as $i | $relations | index($i))] | length ' "$PLAN_FILE") EXTERNAL_TARGETS=$(jq -c --argjson relations "$RELATION_INTERFACES" ' ([.[].collection] | unique) as $included | [ .[] | .schema.fields[] | select(.interface as $i | $relations | index($i)) | .target | select(type == "string" and length > 0) | . as $target | select(($included | index($target)) == null) ] | unique | sort ' "$PLAN_FILE") CATEGORIES=$(jq -c '[.[].categories[]] | unique | sort' "$PLAN_FILE") REAL_CATEGORIES=$(printf '%s' "$CATEGORIES" | jq -c 'map(select(. != "未分类"))') UNCLASSIFIED_COLLECTIONS=$(jq -c ' [.[] | select(.categories | index("未分类")) | .collection] | unique | sort ' "$PLAN_FILE") CATEGORY_RELATION_COUNT=$(jq ' [.[] as $item | $item.categories[] | select(. != "未分类") | [$item.collection, .]] | unique | length ' "$PLAN_FILE") SOURCE_ENV=$(jq -r '.environment // "unknown"' "$MANIFEST_FILE") PLAN_SUMMARY=$(jq -n \ --arg targetEnvironment "$ENV_NAME" \ --arg sourceEnvironment "$SOURCE_ENV" \ --arg inputDir "$INPUT_DIR" \ --argjson collections "$COLLECTION_COUNT" \ --argjson scalarFields "$SCALAR_FIELD_COUNT" \ --argjson relationFields "$RELATION_FIELD_COUNT" \ --argjson categories "$CATEGORIES" \ --argjson categoryRelations "$CATEGORY_RELATION_COUNT" \ --argjson externalRelationTargets "$EXTERNAL_TARGETS" \ --argjson unclassifiedCollections "$UNCLASSIFIED_COLLECTIONS" \ '{ mode:"plan", writes:false, targetEnvironment:$targetEnvironment, sourceEnvironment:$sourceEnvironment, inputDir:$inputDir, profile:"portable", dataSource:"main", collections:$collections, scalarFields:$scalarFields, relationFields:$relationFields, categories:$categories, categoryRelations:$categoryRelations, categoryMembership:"planned-reconciliation", unclassifiedCollections:$unclassifiedCollections, externalRelationTargets:$externalRelationTargets, phases:[ "validate", "apply-collections-and-scalar-fields", "apply-relation-fields", "reconcile-category-memberships", "read-back-and-verify" ] }') if [ "$APPLY" -eq 0 ]; then printf '%s\n' "$PLAN_SUMMARY" | jq . exit 0 fi echo "Validated import plan for $ENV_NAME: $COLLECTION_COUNT Collections, $SCALAR_FIELD_COUNT scalar fields, $RELATION_FIELD_COUNT relation fields" >&2 run_nb() { if ! RAW_OUTPUT=$("$@" 2>&1); then printf '%s\n' "$RAW_OUTPUT" >&2 return 1 fi JSON_OUTPUT=$(printf '%s\n' "$RAW_OUTPUT" | awk ' !started && /^[[:space:]]*[\{\[]/ { started = 1 } started { print } ') if ! printf '%s\n' "$JSON_OUTPUT" | jq -e . >/dev/null 2>&1; then printf '%s\n' "$RAW_OUTPUT" >&2 echo "NocoBase CLI did not return valid JSON" >&2 return 1 fi printf '%s\n' "$JSON_OUTPUT" } load_target_categories() { run_nb nb api resource list \ --resource collectionCategories \ --no-paginate \ --sort sort \ --fields id \ --fields name \ --appends collections \ -e "$ENV_NAME" -y -j } # Reachability and authentication gate before the first write. run_nb nb api data-modeling collections list \ --page 1 --page-size 1 -e "$ENV_NAME" -y -j >/dev/null TARGET_CATEGORIES=$(load_target_categories) if ! printf '%s' "$TARGET_CATEGORIES" | jq -e ' [.data[]?.name] | group_by(.) | all(.[]; length == 1) ' >/dev/null; then echo "Target environment contains duplicate Collection category names" >&2 exit 1 fi TARGET_COUNT=$(printf '%s' "$EXTERNAL_TARGETS" | jq 'length') TARGET_INDEX=0 while [ "$TARGET_INDEX" -lt "$TARGET_COUNT" ]; do TARGET_NAME=$(printf '%s' "$EXTERNAL_TARGETS" | jq -r ".[$TARGET_INDEX]") if ! run_nb nb api data-modeling collections get \ --filter-by-tk "$TARGET_NAME" -e "$ENV_NAME" -y -j >/dev/null; then echo "External relation target is unavailable in $ENV_NAME: $TARGET_NAME" >&2 exit 1 fi TARGET_INDEX=$((TARGET_INDEX + 1)) done make_collection_payload() { PLAN_INDEX=$1 jq --argjson index "$PLAN_INDEX" --argjson relations "$RELATION_INTERFACES" ' def compact_field: . as $field | (($field | del(.settings, .collectionName)) + ($field.settings // {})); .[$index].schema as $schema | ({name:$schema.name, title:$schema.title, template:$schema.template, verify:true, replaceFields:false} + (if $schema | has("description") then {description:$schema.description} else {} end) + (if $schema | has("viewName") then {viewName:$schema.viewName} else {} end) + (if $schema | has("inherits") then {inherits:$schema.inherits} else {} end) + {fields:[ $schema.fields[] | select((.interface as $i | $relations | index($i)) == null) | compact_field ]}) + (((($schema.settings // {}) + ($schema | del( .name, .title, .description, .template, .viewName, .inherits, .replaceFields, .verify, .fields, .settings ))) as $settings | if $settings == {} then {} else {settings:$settings} end)) ' "$PLAN_FILE" } make_relation_payload() { PLAN_INDEX=$1 FIELD_INDEX=$2 jq --argjson planIndex "$PLAN_INDEX" --argjson fieldIndex "$FIELD_INDEX" ' .[$planIndex] as $item | $item.schema.fields[$fieldIndex] as $field | (($field | del(.settings, .collectionName)) + ($field.settings // {})) + {collectionName:$item.collection} ' "$PLAN_FILE" } echo "Phase 1/4: applying Collections and non-relation fields" >&2 INDEX=0 while [ "$INDEX" -lt "$COLLECTION_COUNT" ]; do COLLECTION_NAME=$(jq -r ".[$INDEX].collection" "$PLAN_FILE") PAYLOAD_FILE="$WORK_DIR/collection-$INDEX.json" make_collection_payload "$INDEX" > "$PAYLOAD_FILE" echo " apply $COLLECTION_NAME" >&2 run_nb nb api data-modeling collections apply \ --body-file "$PAYLOAD_FILE" -e "$ENV_NAME" -y -j >/dev/null INDEX=$((INDEX + 1)) done echo "Phase 2/4: applying relation fields" >&2 INDEX=0 while [ "$INDEX" -lt "$COLLECTION_COUNT" ]; do COLLECTION_NAME=$(jq -r ".[$INDEX].collection" "$PLAN_FILE") FIELD_COUNT=$(jq ".[$INDEX].schema.fields | length" "$PLAN_FILE") FIELD_INDEX=0 while [ "$FIELD_INDEX" -lt "$FIELD_COUNT" ]; do INTERFACE=$(jq -r ".[$INDEX].schema.fields[$FIELD_INDEX].interface" "$PLAN_FILE") if printf '%s' "$RELATION_INTERFACES" | jq -e --arg interface "$INTERFACE" 'index($interface) != null' >/dev/null; then FIELD_NAME=$(jq -r ".[$INDEX].schema.fields[$FIELD_INDEX].name" "$PLAN_FILE") PAYLOAD_FILE="$WORK_DIR/relation-$INDEX-$FIELD_INDEX.json" make_relation_payload "$INDEX" "$FIELD_INDEX" > "$PAYLOAD_FILE" echo " apply $COLLECTION_NAME.$FIELD_NAME" >&2 run_nb nb api data-modeling fields apply \ --body-file "$PAYLOAD_FILE" -e "$ENV_NAME" -y -j >/dev/null fi FIELD_INDEX=$((FIELD_INDEX + 1)) done INDEX=$((INDEX + 1)) done echo "Phase 3/4: reconciling Collection category memberships" >&2 REAL_CATEGORY_COUNT=$(printf '%s' "$REAL_CATEGORIES" | jq 'length') CATEGORY_INDEX=0 while [ "$CATEGORY_INDEX" -lt "$REAL_CATEGORY_COUNT" ]; do CATEGORY_NAME=$(printf '%s' "$REAL_CATEGORIES" | jq -r ".[$CATEGORY_INDEX]") if ! printf '%s' "$TARGET_CATEGORIES" | jq -e --arg category "$CATEGORY_NAME" ' any(.data[]?; .name == $category) ' >/dev/null; then CATEGORY_VALUES=$(jq -cn --arg name "$CATEGORY_NAME" '{name:$name}') echo " create category $CATEGORY_NAME" >&2 run_nb nb api resource create \ --resource collectionCategories \ --values "$CATEGORY_VALUES" \ -e "$ENV_NAME" -y -j >/dev/null fi CATEGORY_INDEX=$((CATEGORY_INDEX + 1)) done TARGET_CATEGORIES=$(load_target_categories) IMPORTED_COLLECTIONS=$(jq -c '[.[].collection] | unique | sort' "$PLAN_FILE") TARGET_CATEGORY_COUNT=$(printf '%s' "$TARGET_CATEGORIES" | jq '.data | length') CATEGORY_INDEX=0 while [ "$CATEGORY_INDEX" -lt "$TARGET_CATEGORY_COUNT" ]; do CATEGORY_ID=$(printf '%s' "$TARGET_CATEGORIES" | jq -r ".data[$CATEGORY_INDEX].id") CATEGORY_NAME=$(printf '%s' "$TARGET_CATEGORIES" | jq -r ".data[$CATEGORY_INDEX].name") CURRENT_MEMBERS=$(printf '%s' "$TARGET_CATEGORIES" | jq -c \ ".data[$CATEGORY_INDEX].collections // [] | [.[].name] | unique | sort") DESIRED_IMPORTED=$(jq -c --arg category "$CATEGORY_NAME" ' [.[] | select(.categories | index($category)) | .collection] | unique | sort ' "$PLAN_FILE") DESIRED_MEMBERS=$(jq -cn \ --argjson current "$CURRENT_MEMBERS" \ --argjson imported "$IMPORTED_COLLECTIONS" \ --argjson desiredImported "$DESIRED_IMPORTED" \ '(($current - $imported) + $desiredImported) | unique | sort') if [ "$CURRENT_MEMBERS" != "$DESIRED_MEMBERS" ]; then CATEGORY_VALUES=$(printf '%s' "$DESIRED_MEMBERS" | jq -c '{collections:map({name:.})}') echo " update category $CATEGORY_NAME" >&2 # Repeat the single association flag so nb CLI serializes it as an array. # A single query value is parsed by NocoBase as a string and causes # UpdateGuard to fail with "list.filter is not a function". run_nb nb api resource update \ --resource collectionCategories \ --filter-by-tk "$CATEGORY_ID" \ --values "$CATEGORY_VALUES" \ --update-association-values collections \ --update-association-values collections \ -e "$ENV_NAME" -y -j >/dev/null fi CATEGORY_INDEX=$((CATEGORY_INDEX + 1)) done portable_schema() { jq ' .data as $collection | (($collection.fields // []) | map( . as $field | (($field | del(.settings)) + ($field.settings // {})) )) as $fields | [$fields[] | select(.interface == "m2o" or .interface == "o2m" or .interface == "m2m" or .interface == "o2o") | .foreignKey | select(. != null)] as $foreignKeys | ($collection.template // "general") as $template | ($collection | del(.key, .fields, .verify, .category)) + { fields: [ $fields[] | . as $field | select(["id", "createdAt", "createdBy", "updatedAt", "updatedBy", "createdById", "updatedById"] | index($field.name) | not) | select( if $template == "file" then ["title", "filename", "extname", "size", "mimetype", "path", "url", "preview", "storage", "storageId", "meta"] | index($field.name) | not elif $template == "tree" then ["parentId", "parent", "children"] | index($field.name) | not else true end ) | select( ($foreignKeys | index($field.name) | not) or ($field.interface == "m2o" or $field.interface == "o2m" or $field.interface == "m2m" or $field.interface == "o2o") ) | del(.key, .collectionName, .parentKey, .reverseKey, .possibleTypes) | walk( if type == "object" and has("rules") and (.rules | type) == "array" then .rules |= map(del(.key)) else . end ) ] | sort_by(.name) } ' } echo "Phase 4/4: reading back and verifying" >&2 INDEX=0 while [ "$INDEX" -lt "$COLLECTION_COUNT" ]; do COLLECTION_NAME=$(jq -r ".[$INDEX].collection" "$PLAN_FILE") EXPECTED=$(jq -c ".[$INDEX].schema" "$PLAN_FILE") READBACK=$(run_nb nb api data-modeling collections get \ --filter-by-tk "$COLLECTION_NAME" --appends fields \ -e "$ENV_NAME" -y -j) ACTUAL=$(printf '%s' "$READBACK" | portable_schema | jq -c .) if ! jq -en --argjson expected "$EXPECTED" --argjson actual "$ACTUAL" ' $actual | contains($expected | del(.fields)) ' >/dev/null; then jq -cn --arg collection "$COLLECTION_NAME" \ --argjson expected "$EXPECTED" --argjson actual "$ACTUAL" \ '{collection:$collection, scope:"collection", expected:($expected|del(.fields)), actual:($actual|del(.fields))}' \ >> "$VERIFY_FILE" fi jq -cn --arg collection "$COLLECTION_NAME" \ --argjson expected "$EXPECTED" --argjson actual "$ACTUAL" ' [ $expected.fields[] as $field | ($actual.fields | map(select(.name == $field.name)) | first // null) as $found | select($found == null or (($found | contains($field)) | not)) | {collection:$collection, scope:"field", field:$field.name, expected:$field, actual:$found} ][] ' >> "$VERIFY_FILE" INDEX=$((INDEX + 1)) done CATEGORY_READBACK=$(load_target_categories) EXPECTED_CATEGORY_RELATIONS=$(jq -c ' [ .[] as $item | $item.categories[] | select(. != "未分类") | {category:., collection:$item.collection} ] | unique | sort_by(.category, .collection) ' "$PLAN_FILE") ACTUAL_CATEGORY_RELATIONS=$(printf '%s' "$CATEGORY_READBACK" | jq -c \ --argjson imported "$IMPORTED_COLLECTIONS" ' [ .data[]? as $category | $category.collections[]? | select(.name as $name | $imported | index($name)) | {category:$category.name, collection:.name} ] | unique | sort_by(.category, .collection) ') if [ "$EXPECTED_CATEGORY_RELATIONS" != "$ACTUAL_CATEGORY_RELATIONS" ]; then jq -cn \ --argjson expected "$EXPECTED_CATEGORY_RELATIONS" \ --argjson actual "$ACTUAL_CATEGORY_RELATIONS" \ '{scope:"collection-category-memberships", expected:$expected, actual:$actual}' \ >> "$VERIFY_FILE" fi if [ -s "$VERIFY_FILE" ]; then echo "Import read-back verification failed:" >&2 jq -s . "$VERIFY_FILE" >&2 exit 1 fi jq -n \ --arg targetEnvironment "$ENV_NAME" \ --arg sourceEnvironment "$SOURCE_ENV" \ --arg inputDir "$INPUT_DIR" \ --argjson collections "$COLLECTION_COUNT" \ --argjson scalarFields "$SCALAR_FIELD_COUNT" \ --argjson relationFields "$RELATION_FIELD_COUNT" \ --argjson categories "$CATEGORIES" \ --argjson categoryRelations "$CATEGORY_RELATION_COUNT" \ --argjson unclassifiedCollections "$UNCLASSIFIED_COLLECTIONS" \ '{ mode:"applied", writes:true, verified:true, targetEnvironment:$targetEnvironment, sourceEnvironment:$sourceEnvironment, inputDir:$inputDir, profile:"portable", dataSource:"main", collections:$collections, scalarFields:$scalarFields, relationFields:$relationFields, categories:$categories, categoryRelations:$categoryRelations, categoryMembership:"verified", unclassifiedCollections:$unclassifiedCollections }'