Ethan
9dc20b03e5
Normalize portable field settings during export and import, serialize collection category association updates as arrays, and ignore generated exports and macOS metadata.
571 lines
20 KiB
Bash
Executable File
571 lines
20 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
./script/import-collection-schema.sh \
|
|
--env <target environment> \
|
|
--input-dir <export directory> \
|
|
[--apply --confirm-env <same target environment>]
|
|
|
|
Options:
|
|
--env Required target NocoBase CLI environment name.
|
|
--input-dir Required directory produced by export-collection-schema.sh.
|
|
--apply Apply the validated plan. Default: plan only, no writes.
|
|
--confirm-env Required with --apply and must exactly equal --env.
|
|
-h, --help Show this help.
|
|
|
|
Behavior:
|
|
- Accepts only portable schemas for the main data source.
|
|
- Validates manifest.json and every referenced schema before any write.
|
|
- Applies collection shells and non-relation fields first, then relations.
|
|
- Reads every imported collection back and verifies the declared schema.
|
|
- Creates missing Collection categories and reconciles category memberships.
|
|
- Treats the exported pseudo-category "未分类" as no category membership.
|
|
- Does not delete extra fields already present in the target environment.
|
|
|
|
Plan only:
|
|
./script/import-collection-schema.sh \
|
|
--env mdm-uat \
|
|
--input-dir ./exports/reference-schema
|
|
|
|
Apply:
|
|
./script/import-collection-schema.sh \
|
|
--env mdm-uat \
|
|
--input-dir ./exports/reference-schema \
|
|
--apply \
|
|
--confirm-env mdm-uat
|
|
EOF
|
|
}
|
|
|
|
ENV_NAME=
|
|
INPUT_DIR=
|
|
APPLY=0
|
|
CONFIRM_ENV=
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--env)
|
|
ENV_NAME=${2:?--env requires a value}
|
|
shift 2
|
|
;;
|
|
--input-dir)
|
|
INPUT_DIR=${2:?--input-dir requires a value}
|
|
shift 2
|
|
;;
|
|
--apply)
|
|
APPLY=1
|
|
shift
|
|
;;
|
|
--confirm-env)
|
|
CONFIRM_ENV=${2:?--confirm-env requires a value}
|
|
shift 2
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown argument: $1" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[ -n "$ENV_NAME" ] || { echo "--env is required" >&2; exit 2; }
|
|
[ -n "$INPUT_DIR" ] || { echo "--input-dir is required" >&2; exit 2; }
|
|
[ -d "$INPUT_DIR" ] || { echo "Input directory does not exist: $INPUT_DIR" >&2; exit 1; }
|
|
command -v nb >/dev/null 2>&1 || { echo "nb CLI is required" >&2; exit 2; }
|
|
command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2; exit 2; }
|
|
|
|
if [ "$APPLY" -eq 0 ] && [ -n "$CONFIRM_ENV" ]; then
|
|
echo "--confirm-env is only valid together with --apply" >&2
|
|
exit 2
|
|
fi
|
|
if [ "$APPLY" -eq 1 ] && [ "$CONFIRM_ENV" != "$ENV_NAME" ]; then
|
|
echo "--apply requires --confirm-env to exactly equal --env ($ENV_NAME)" >&2
|
|
exit 2
|
|
fi
|
|
|
|
INPUT_DIR=$(CDPATH= cd -- "$INPUT_DIR" && pwd)
|
|
MANIFEST_FILE="$INPUT_DIR/manifest.json"
|
|
[ -f "$MANIFEST_FILE" ] && [ ! -L "$MANIFEST_FILE" ] || {
|
|
echo "Regular manifest.json not found in: $INPUT_DIR" >&2
|
|
exit 1
|
|
}
|
|
|
|
if ! jq -e '
|
|
type == "object"
|
|
and .profile == "portable"
|
|
and .dataSource == "main"
|
|
and (.files | type == "array" and length > 0)
|
|
and all(.files[];
|
|
.dataSource == "main"
|
|
and (.category | type == "string" and length > 0)
|
|
and (.collection | type == "string" and length > 0)
|
|
and (.file | type == "string" and length > 0)
|
|
and (.file | startswith("/") | not)
|
|
and (.file | contains("\\") | not)
|
|
and (.file | split("/") | all(. != "" and . != "." and . != ".."))
|
|
)
|
|
and (([.files[].file] | unique | length) == (.files | length))
|
|
and ((has("exportedFiles") | not) or .exportedFiles == (.files | length))
|
|
' "$MANIFEST_FILE" >/dev/null; then
|
|
echo "Invalid manifest.json: portable main-data-source export required" >&2
|
|
exit 1
|
|
fi
|
|
|
|
WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/import-collection-schema.XXXXXX")
|
|
ROWS_FILE="$WORK_DIR/schema-rows.jsonl"
|
|
PLAN_FILE="$WORK_DIR/plan.json"
|
|
VERIFY_FILE="$WORK_DIR/verification-errors.jsonl"
|
|
: > "$ROWS_FILE"
|
|
: > "$VERIFY_FILE"
|
|
trap 'rm -rf "$WORK_DIR"' EXIT HUP INT TERM
|
|
|
|
FILE_COUNT=$(jq '.files | length' "$MANIFEST_FILE")
|
|
INDEX=0
|
|
while [ "$INDEX" -lt "$FILE_COUNT" ]; do
|
|
ITEM=$(jq -c ".files[$INDEX]" "$MANIFEST_FILE")
|
|
RELATIVE_FILE=$(printf '%s' "$ITEM" | jq -r '.file')
|
|
COLLECTION_NAME=$(printf '%s' "$ITEM" | jq -r '.collection')
|
|
CATEGORY_NAME=$(printf '%s' "$ITEM" | jq -r '.category')
|
|
SCHEMA_FILE="$INPUT_DIR/$RELATIVE_FILE"
|
|
|
|
[ -f "$SCHEMA_FILE" ] && [ ! -L "$SCHEMA_FILE" ] || {
|
|
echo "Schema file is missing or is not a regular file: $RELATIVE_FILE" >&2
|
|
exit 1
|
|
}
|
|
if ! jq -e --arg collection "$COLLECTION_NAME" '
|
|
type == "object"
|
|
and .name == $collection
|
|
and (.title | type == "string" and length > 0)
|
|
and (.template | type == "string" and length > 0)
|
|
and (.fields | type == "array")
|
|
and all(.fields[];
|
|
type == "object"
|
|
and (.name | type == "string" and length > 0)
|
|
and (.interface | type == "string" and length > 0)
|
|
)
|
|
and (([.fields[].name] | unique | length) == (.fields | length))
|
|
' "$SCHEMA_FILE" >/dev/null; then
|
|
echo "Invalid Collection schema: $RELATIVE_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq -c \
|
|
--arg collection "$COLLECTION_NAME" \
|
|
--arg category "$CATEGORY_NAME" \
|
|
--arg file "$RELATIVE_FILE" \
|
|
'{collection:$collection, category:$category, file:$file, schema:.}' \
|
|
"$SCHEMA_FILE" >> "$ROWS_FILE"
|
|
INDEX=$((INDEX + 1))
|
|
done
|
|
|
|
if ! jq -se '
|
|
sort_by(.collection)
|
|
| group_by(.collection)
|
|
| all(.[]; ([.[].schema] | unique | length) == 1)
|
|
' "$ROWS_FILE" >/dev/null; then
|
|
echo "One Collection is represented by conflicting schema files" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq -s '
|
|
sort_by(.collection)
|
|
| group_by(.collection)
|
|
| map({
|
|
collection: .[0].collection,
|
|
categories: ([.[].category] | unique | sort),
|
|
files: ([.[].file] | unique | sort),
|
|
schema: .[0].schema
|
|
})
|
|
' "$ROWS_FILE" > "$PLAN_FILE"
|
|
|
|
COLLECTION_COUNT=$(jq 'length' "$PLAN_FILE")
|
|
if ! jq -e --argjson count "$COLLECTION_COUNT" '
|
|
(has("exportedCollections") | not) or .exportedCollections == $count
|
|
' "$MANIFEST_FILE" >/dev/null; then
|
|
echo "manifest.json exportedCollections does not match the schemas" >&2
|
|
exit 1
|
|
fi
|
|
|
|
RELATION_INTERFACES='["m2o","o2m","m2m","o2o"]'
|
|
SCALAR_FIELD_COUNT=$(jq --argjson relations "$RELATION_INTERFACES" '
|
|
[.[] | .schema.fields[] | select((.interface as $i | $relations | index($i)) == null)] | length
|
|
' "$PLAN_FILE")
|
|
RELATION_FIELD_COUNT=$(jq --argjson relations "$RELATION_INTERFACES" '
|
|
[.[] | .schema.fields[] | select(.interface as $i | $relations | index($i))] | length
|
|
' "$PLAN_FILE")
|
|
EXTERNAL_TARGETS=$(jq -c --argjson relations "$RELATION_INTERFACES" '
|
|
([.[].collection] | unique) as $included
|
|
| [
|
|
.[] | .schema.fields[]
|
|
| select(.interface as $i | $relations | index($i))
|
|
| .target
|
|
| select(type == "string" and length > 0)
|
|
| . as $target
|
|
| select(($included | index($target)) == null)
|
|
] | unique | sort
|
|
' "$PLAN_FILE")
|
|
CATEGORIES=$(jq -c '[.[].categories[]] | unique | sort' "$PLAN_FILE")
|
|
REAL_CATEGORIES=$(printf '%s' "$CATEGORIES" | jq -c 'map(select(. != "未分类"))')
|
|
UNCLASSIFIED_COLLECTIONS=$(jq -c '
|
|
[.[] | select(.categories | index("未分类")) | .collection] | unique | sort
|
|
' "$PLAN_FILE")
|
|
CATEGORY_RELATION_COUNT=$(jq '
|
|
[.[] as $item | $item.categories[] | select(. != "未分类") | [$item.collection, .]]
|
|
| unique | length
|
|
' "$PLAN_FILE")
|
|
SOURCE_ENV=$(jq -r '.environment // "unknown"' "$MANIFEST_FILE")
|
|
|
|
PLAN_SUMMARY=$(jq -n \
|
|
--arg targetEnvironment "$ENV_NAME" \
|
|
--arg sourceEnvironment "$SOURCE_ENV" \
|
|
--arg inputDir "$INPUT_DIR" \
|
|
--argjson collections "$COLLECTION_COUNT" \
|
|
--argjson scalarFields "$SCALAR_FIELD_COUNT" \
|
|
--argjson relationFields "$RELATION_FIELD_COUNT" \
|
|
--argjson categories "$CATEGORIES" \
|
|
--argjson categoryRelations "$CATEGORY_RELATION_COUNT" \
|
|
--argjson externalRelationTargets "$EXTERNAL_TARGETS" \
|
|
--argjson unclassifiedCollections "$UNCLASSIFIED_COLLECTIONS" \
|
|
'{
|
|
mode:"plan",
|
|
writes:false,
|
|
targetEnvironment:$targetEnvironment,
|
|
sourceEnvironment:$sourceEnvironment,
|
|
inputDir:$inputDir,
|
|
profile:"portable",
|
|
dataSource:"main",
|
|
collections:$collections,
|
|
scalarFields:$scalarFields,
|
|
relationFields:$relationFields,
|
|
categories:$categories,
|
|
categoryRelations:$categoryRelations,
|
|
categoryMembership:"planned-reconciliation",
|
|
unclassifiedCollections:$unclassifiedCollections,
|
|
externalRelationTargets:$externalRelationTargets,
|
|
phases:[
|
|
"validate",
|
|
"apply-collections-and-scalar-fields",
|
|
"apply-relation-fields",
|
|
"reconcile-category-memberships",
|
|
"read-back-and-verify"
|
|
]
|
|
}')
|
|
|
|
if [ "$APPLY" -eq 0 ]; then
|
|
printf '%s\n' "$PLAN_SUMMARY" | jq .
|
|
exit 0
|
|
fi
|
|
|
|
echo "Validated import plan for $ENV_NAME: $COLLECTION_COUNT Collections, $SCALAR_FIELD_COUNT scalar fields, $RELATION_FIELD_COUNT relation fields" >&2
|
|
|
|
run_nb() {
|
|
if ! RAW_OUTPUT=$("$@" 2>&1); then
|
|
printf '%s\n' "$RAW_OUTPUT" >&2
|
|
return 1
|
|
fi
|
|
JSON_OUTPUT=$(printf '%s\n' "$RAW_OUTPUT" | awk '
|
|
!started && /^[[:space:]]*[\{\[]/ { started = 1 }
|
|
started { print }
|
|
')
|
|
if ! printf '%s\n' "$JSON_OUTPUT" | jq -e . >/dev/null 2>&1; then
|
|
printf '%s\n' "$RAW_OUTPUT" >&2
|
|
echo "NocoBase CLI did not return valid JSON" >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$JSON_OUTPUT"
|
|
}
|
|
|
|
load_target_categories() {
|
|
run_nb nb api resource list \
|
|
--resource collectionCategories \
|
|
--no-paginate \
|
|
--sort sort \
|
|
--fields id \
|
|
--fields name \
|
|
--appends collections \
|
|
-e "$ENV_NAME" -y -j
|
|
}
|
|
|
|
# Reachability and authentication gate before the first write.
|
|
run_nb nb api data-modeling collections list \
|
|
--page 1 --page-size 1 -e "$ENV_NAME" -y -j >/dev/null
|
|
|
|
TARGET_CATEGORIES=$(load_target_categories)
|
|
if ! printf '%s' "$TARGET_CATEGORIES" | jq -e '
|
|
[.data[]?.name] | group_by(.) | all(.[]; length == 1)
|
|
' >/dev/null; then
|
|
echo "Target environment contains duplicate Collection category names" >&2
|
|
exit 1
|
|
fi
|
|
|
|
TARGET_COUNT=$(printf '%s' "$EXTERNAL_TARGETS" | jq 'length')
|
|
TARGET_INDEX=0
|
|
while [ "$TARGET_INDEX" -lt "$TARGET_COUNT" ]; do
|
|
TARGET_NAME=$(printf '%s' "$EXTERNAL_TARGETS" | jq -r ".[$TARGET_INDEX]")
|
|
if ! run_nb nb api data-modeling collections get \
|
|
--filter-by-tk "$TARGET_NAME" -e "$ENV_NAME" -y -j >/dev/null; then
|
|
echo "External relation target is unavailable in $ENV_NAME: $TARGET_NAME" >&2
|
|
exit 1
|
|
fi
|
|
TARGET_INDEX=$((TARGET_INDEX + 1))
|
|
done
|
|
|
|
make_collection_payload() {
|
|
PLAN_INDEX=$1
|
|
jq --argjson index "$PLAN_INDEX" --argjson relations "$RELATION_INTERFACES" '
|
|
def compact_field:
|
|
. as $field
|
|
| (($field | del(.settings, .collectionName)) + ($field.settings // {}));
|
|
.[$index].schema as $schema
|
|
| ({name:$schema.name, title:$schema.title, template:$schema.template, verify:true, replaceFields:false}
|
|
+ (if $schema | has("description") then {description:$schema.description} else {} end)
|
|
+ (if $schema | has("viewName") then {viewName:$schema.viewName} else {} end)
|
|
+ (if $schema | has("inherits") then {inherits:$schema.inherits} else {} end)
|
|
+ {fields:[
|
|
$schema.fields[]
|
|
| select((.interface as $i | $relations | index($i)) == null)
|
|
| compact_field
|
|
]})
|
|
+ (((($schema.settings // {}) + ($schema | del(
|
|
.name, .title, .description, .template, .viewName, .inherits,
|
|
.replaceFields, .verify, .fields, .settings
|
|
))) as $settings
|
|
| if $settings == {} then {} else {settings:$settings} end))
|
|
' "$PLAN_FILE"
|
|
}
|
|
|
|
make_relation_payload() {
|
|
PLAN_INDEX=$1
|
|
FIELD_INDEX=$2
|
|
jq --argjson planIndex "$PLAN_INDEX" --argjson fieldIndex "$FIELD_INDEX" '
|
|
.[$planIndex] as $item
|
|
| $item.schema.fields[$fieldIndex] as $field
|
|
| (($field | del(.settings, .collectionName)) + ($field.settings // {}))
|
|
+ {collectionName:$item.collection}
|
|
' "$PLAN_FILE"
|
|
}
|
|
|
|
echo "Phase 1/4: applying Collections and non-relation fields" >&2
|
|
INDEX=0
|
|
while [ "$INDEX" -lt "$COLLECTION_COUNT" ]; do
|
|
COLLECTION_NAME=$(jq -r ".[$INDEX].collection" "$PLAN_FILE")
|
|
PAYLOAD_FILE="$WORK_DIR/collection-$INDEX.json"
|
|
make_collection_payload "$INDEX" > "$PAYLOAD_FILE"
|
|
echo " apply $COLLECTION_NAME" >&2
|
|
run_nb nb api data-modeling collections apply \
|
|
--body-file "$PAYLOAD_FILE" -e "$ENV_NAME" -y -j >/dev/null
|
|
INDEX=$((INDEX + 1))
|
|
done
|
|
|
|
echo "Phase 2/4: applying relation fields" >&2
|
|
INDEX=0
|
|
while [ "$INDEX" -lt "$COLLECTION_COUNT" ]; do
|
|
COLLECTION_NAME=$(jq -r ".[$INDEX].collection" "$PLAN_FILE")
|
|
FIELD_COUNT=$(jq ".[$INDEX].schema.fields | length" "$PLAN_FILE")
|
|
FIELD_INDEX=0
|
|
while [ "$FIELD_INDEX" -lt "$FIELD_COUNT" ]; do
|
|
INTERFACE=$(jq -r ".[$INDEX].schema.fields[$FIELD_INDEX].interface" "$PLAN_FILE")
|
|
if printf '%s' "$RELATION_INTERFACES" | jq -e --arg interface "$INTERFACE" 'index($interface) != null' >/dev/null; then
|
|
FIELD_NAME=$(jq -r ".[$INDEX].schema.fields[$FIELD_INDEX].name" "$PLAN_FILE")
|
|
PAYLOAD_FILE="$WORK_DIR/relation-$INDEX-$FIELD_INDEX.json"
|
|
make_relation_payload "$INDEX" "$FIELD_INDEX" > "$PAYLOAD_FILE"
|
|
echo " apply $COLLECTION_NAME.$FIELD_NAME" >&2
|
|
run_nb nb api data-modeling fields apply \
|
|
--body-file "$PAYLOAD_FILE" -e "$ENV_NAME" -y -j >/dev/null
|
|
fi
|
|
FIELD_INDEX=$((FIELD_INDEX + 1))
|
|
done
|
|
INDEX=$((INDEX + 1))
|
|
done
|
|
|
|
echo "Phase 3/4: reconciling Collection category memberships" >&2
|
|
REAL_CATEGORY_COUNT=$(printf '%s' "$REAL_CATEGORIES" | jq 'length')
|
|
CATEGORY_INDEX=0
|
|
while [ "$CATEGORY_INDEX" -lt "$REAL_CATEGORY_COUNT" ]; do
|
|
CATEGORY_NAME=$(printf '%s' "$REAL_CATEGORIES" | jq -r ".[$CATEGORY_INDEX]")
|
|
if ! printf '%s' "$TARGET_CATEGORIES" | jq -e --arg category "$CATEGORY_NAME" '
|
|
any(.data[]?; .name == $category)
|
|
' >/dev/null; then
|
|
CATEGORY_VALUES=$(jq -cn --arg name "$CATEGORY_NAME" '{name:$name}')
|
|
echo " create category $CATEGORY_NAME" >&2
|
|
run_nb nb api resource create \
|
|
--resource collectionCategories \
|
|
--values "$CATEGORY_VALUES" \
|
|
-e "$ENV_NAME" -y -j >/dev/null
|
|
fi
|
|
CATEGORY_INDEX=$((CATEGORY_INDEX + 1))
|
|
done
|
|
|
|
TARGET_CATEGORIES=$(load_target_categories)
|
|
IMPORTED_COLLECTIONS=$(jq -c '[.[].collection] | unique | sort' "$PLAN_FILE")
|
|
TARGET_CATEGORY_COUNT=$(printf '%s' "$TARGET_CATEGORIES" | jq '.data | length')
|
|
CATEGORY_INDEX=0
|
|
while [ "$CATEGORY_INDEX" -lt "$TARGET_CATEGORY_COUNT" ]; do
|
|
CATEGORY_ID=$(printf '%s' "$TARGET_CATEGORIES" | jq -r ".data[$CATEGORY_INDEX].id")
|
|
CATEGORY_NAME=$(printf '%s' "$TARGET_CATEGORIES" | jq -r ".data[$CATEGORY_INDEX].name")
|
|
CURRENT_MEMBERS=$(printf '%s' "$TARGET_CATEGORIES" | jq -c \
|
|
".data[$CATEGORY_INDEX].collections // [] | [.[].name] | unique | sort")
|
|
DESIRED_IMPORTED=$(jq -c --arg category "$CATEGORY_NAME" '
|
|
[.[] | select(.categories | index($category)) | .collection] | unique | sort
|
|
' "$PLAN_FILE")
|
|
DESIRED_MEMBERS=$(jq -cn \
|
|
--argjson current "$CURRENT_MEMBERS" \
|
|
--argjson imported "$IMPORTED_COLLECTIONS" \
|
|
--argjson desiredImported "$DESIRED_IMPORTED" \
|
|
'(($current - $imported) + $desiredImported) | unique | sort')
|
|
|
|
if [ "$CURRENT_MEMBERS" != "$DESIRED_MEMBERS" ]; then
|
|
CATEGORY_VALUES=$(printf '%s' "$DESIRED_MEMBERS" | jq -c '{collections:map({name:.})}')
|
|
echo " update category $CATEGORY_NAME" >&2
|
|
# Repeat the single association flag so nb CLI serializes it as an array.
|
|
# A single query value is parsed by NocoBase as a string and causes
|
|
# UpdateGuard to fail with "list.filter is not a function".
|
|
run_nb nb api resource update \
|
|
--resource collectionCategories \
|
|
--filter-by-tk "$CATEGORY_ID" \
|
|
--values "$CATEGORY_VALUES" \
|
|
--update-association-values collections \
|
|
--update-association-values collections \
|
|
-e "$ENV_NAME" -y -j >/dev/null
|
|
fi
|
|
CATEGORY_INDEX=$((CATEGORY_INDEX + 1))
|
|
done
|
|
|
|
portable_schema() {
|
|
jq '
|
|
.data as $collection
|
|
| (($collection.fields // []) | map(
|
|
. as $field
|
|
| (($field | del(.settings)) + ($field.settings // {}))
|
|
)) as $fields
|
|
| [$fields[]
|
|
| select(.interface == "m2o" or .interface == "o2m" or .interface == "m2m" or .interface == "o2o")
|
|
| .foreignKey
|
|
| select(. != null)] as $foreignKeys
|
|
| ($collection.template // "general") as $template
|
|
| ($collection | del(.key, .fields, .verify, .category))
|
|
+ {
|
|
fields: [
|
|
$fields[]
|
|
| . as $field
|
|
| select(["id", "createdAt", "createdBy", "updatedAt", "updatedBy", "createdById", "updatedById"] | index($field.name) | not)
|
|
| select(
|
|
if $template == "file" then
|
|
["title", "filename", "extname", "size", "mimetype", "path", "url", "preview", "storage", "storageId", "meta"]
|
|
| index($field.name) | not
|
|
elif $template == "tree" then
|
|
["parentId", "parent", "children"] | index($field.name) | not
|
|
else true end
|
|
)
|
|
| select(
|
|
($foreignKeys | index($field.name) | not)
|
|
or ($field.interface == "m2o" or $field.interface == "o2m" or $field.interface == "m2m" or $field.interface == "o2o")
|
|
)
|
|
| del(.key, .collectionName, .parentKey, .reverseKey, .possibleTypes)
|
|
| walk(
|
|
if type == "object" and has("rules") and (.rules | type) == "array" then
|
|
.rules |= map(del(.key))
|
|
else . end
|
|
)
|
|
] | sort_by(.name)
|
|
}
|
|
'
|
|
}
|
|
|
|
echo "Phase 4/4: reading back and verifying" >&2
|
|
INDEX=0
|
|
while [ "$INDEX" -lt "$COLLECTION_COUNT" ]; do
|
|
COLLECTION_NAME=$(jq -r ".[$INDEX].collection" "$PLAN_FILE")
|
|
EXPECTED=$(jq -c ".[$INDEX].schema" "$PLAN_FILE")
|
|
READBACK=$(run_nb nb api data-modeling collections get \
|
|
--filter-by-tk "$COLLECTION_NAME" --appends fields \
|
|
-e "$ENV_NAME" -y -j)
|
|
ACTUAL=$(printf '%s' "$READBACK" | portable_schema | jq -c .)
|
|
|
|
if ! jq -en --argjson expected "$EXPECTED" --argjson actual "$ACTUAL" '
|
|
$actual | contains($expected | del(.fields))
|
|
' >/dev/null; then
|
|
jq -cn --arg collection "$COLLECTION_NAME" \
|
|
--argjson expected "$EXPECTED" --argjson actual "$ACTUAL" \
|
|
'{collection:$collection, scope:"collection", expected:($expected|del(.fields)), actual:($actual|del(.fields))}' \
|
|
>> "$VERIFY_FILE"
|
|
fi
|
|
|
|
jq -cn --arg collection "$COLLECTION_NAME" \
|
|
--argjson expected "$EXPECTED" --argjson actual "$ACTUAL" '
|
|
[
|
|
$expected.fields[] as $field
|
|
| ($actual.fields | map(select(.name == $field.name)) | first // null) as $found
|
|
| select($found == null or (($found | contains($field)) | not))
|
|
| {collection:$collection, scope:"field", field:$field.name, expected:$field, actual:$found}
|
|
][]
|
|
' >> "$VERIFY_FILE"
|
|
INDEX=$((INDEX + 1))
|
|
done
|
|
|
|
CATEGORY_READBACK=$(load_target_categories)
|
|
EXPECTED_CATEGORY_RELATIONS=$(jq -c '
|
|
[
|
|
.[] as $item
|
|
| $item.categories[]
|
|
| select(. != "未分类")
|
|
| {category:., collection:$item.collection}
|
|
] | unique | sort_by(.category, .collection)
|
|
' "$PLAN_FILE")
|
|
ACTUAL_CATEGORY_RELATIONS=$(printf '%s' "$CATEGORY_READBACK" | jq -c \
|
|
--argjson imported "$IMPORTED_COLLECTIONS" '
|
|
[
|
|
.data[]? as $category
|
|
| $category.collections[]?
|
|
| select(.name as $name | $imported | index($name))
|
|
| {category:$category.name, collection:.name}
|
|
] | unique | sort_by(.category, .collection)
|
|
')
|
|
if [ "$EXPECTED_CATEGORY_RELATIONS" != "$ACTUAL_CATEGORY_RELATIONS" ]; then
|
|
jq -cn \
|
|
--argjson expected "$EXPECTED_CATEGORY_RELATIONS" \
|
|
--argjson actual "$ACTUAL_CATEGORY_RELATIONS" \
|
|
'{scope:"collection-category-memberships", expected:$expected, actual:$actual}' \
|
|
>> "$VERIFY_FILE"
|
|
fi
|
|
|
|
if [ -s "$VERIFY_FILE" ]; then
|
|
echo "Import read-back verification failed:" >&2
|
|
jq -s . "$VERIFY_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
jq -n \
|
|
--arg targetEnvironment "$ENV_NAME" \
|
|
--arg sourceEnvironment "$SOURCE_ENV" \
|
|
--arg inputDir "$INPUT_DIR" \
|
|
--argjson collections "$COLLECTION_COUNT" \
|
|
--argjson scalarFields "$SCALAR_FIELD_COUNT" \
|
|
--argjson relationFields "$RELATION_FIELD_COUNT" \
|
|
--argjson categories "$CATEGORIES" \
|
|
--argjson categoryRelations "$CATEGORY_RELATION_COUNT" \
|
|
--argjson unclassifiedCollections "$UNCLASSIFIED_COLLECTIONS" \
|
|
'{
|
|
mode:"applied",
|
|
writes:true,
|
|
verified:true,
|
|
targetEnvironment:$targetEnvironment,
|
|
sourceEnvironment:$sourceEnvironment,
|
|
inputDir:$inputDir,
|
|
profile:"portable",
|
|
dataSource:"main",
|
|
collections:$collections,
|
|
scalarFields:$scalarFields,
|
|
relationFields:$relationFields,
|
|
categories:$categories,
|
|
categoryRelations:$categoryRelations,
|
|
categoryMembership:"verified",
|
|
unclassifiedCollections:$unclassifiedCollections
|
|
}'
|