Reputation: 1
I have a shell scripts that export mongodb and timescale databases from one namespace to another, I call it this way:
.github/scripts/update-db.sh -mdb ${{ inputs.refresh_mongodb }} -tsdb ${{ inputs.refresh_timescaledb }} -src ${{ inputs.source_namespace }} -dst ${{ inputs.destination_namespace }}
It's called via a github workflow. I am getting the databases to be exported but the problem, is that in the destination databases the password changes! it becomes the password of the source database! as if the password is being exported and overwritten !
I don't want this behaviour, i want only to export the data.
Here's my script:
#!/bin/bash
# Vvariables
export ANONYMIZE_SCRIPT=".github/scripts/anonymize.js"
export UPDATE_TABLE_OWNER="DO \$\$
DECLARE
table_name_var text;
BEGIN
FOR table_name_var IN (SELECT table_name FROM information_schema.tables WHERE table_schema = 'mtngeconnect')
LOOP
EXECUTE 'ALTER TABLE mtngeconnect.' || table_name_var || ' OWNER TO quantumleap';
END LOOP;
END \$\$;"
# Default variables
mdb_option="y"
tsdb_option="y"
src_namespace="aap"
dst_namespace="ppr"
mdb_executed=false
tsdb_executed=false
# Options
while [ "$#" -gt 0 ]; do
case "$1" in
-mdb)
shift
mdb_option="$1"
;;
-tsdb)
shift
tsdb_option="$1"
;;
-src)
shift
src_namespace="$1"
;;
-dst)
shift
dst_namespace="$1"
;;
-help)
echo "Script de sauvegarde et restauration des BDD pour la préproduction"
echo "Usage:"
echo "-mdb y : backup restauration MongoDB"
echo "-tsdb y : backup restauration TimescaleDB"
echo "-src <namespace> : namespace source"
echo "-dst <namespace> : namespace destination"
echo "-help : afficher l'aide"
exit 1
;;
*)
echo "Option inconnue: $1"
exit 1
;;
esac
shift
done
function scale_pods {
if { [ "$mdb_option" == "y" ] && [ "$mdb_executed" == false ]; } || { [ "$tsdb_option" == "y" ] && [ "$tsdb_executed" == false ]; }
then
echo "Scaling down all deployments in namespace $dst_namespace"
for object in $(kubectl get deployment -n "$dst_namespace" | grep -v 'NAME' | awk '{print $1}')
do
kubectl scale --replicas=0 deployment "$object" -n "$dst_namespace"
done
fi
if [ "$mdb_executed" == true ] || [ "$tsdb_executed" == true ]
then
echo "Scaling up all deployments in namespace $dst_namespace"
for object in $(kubectl get deployment -n "$dst_namespace" | grep -v 'NAME' | awk '{print $1}')
do
kubectl scale --replicas=1 deployment "$object" -n "$dst_namespace"
done
fi
}
# Function for mdb option
function process_mdb {
if [ "$mdb_option" == "y" ]; then
echo "Traitement MDB..."
##MongoDB
if ! grep mongodb /etc/hosts > /dev/null
then
echo "127.0.0.1 mongodb-secondary mongodb-primary rsmart-mongodb-0 rsmart-mongodb-1 rsmart-mongodb-2" >> /etc/hosts
fi
## install mongo tools
if ! command -v mongodump &> /dev/null
then
echo "Install mongodb tools"
wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.4.deb
sudo apt install ./mongodb-database-tools-*.deb
fi
## Test if ANONYMIZE_SCRIPT
if [ ! -f $ANONYMIZE_SCRIPT ]
then
echo "ERROR: $ANONYMIZE_SCRIPT doesn't exist"
exit 1
fi
## Get MongoDB pass
echo "Get MongoDB password from namespace: $src_namespace"
src_mongodb_password=$(kubectl exec -it svc/rsmart-mongodb -n "$src_namespace" -c mongodb -- env | grep MONGODB_ROOT_PASSWORD | cut -d '=' -f 2)
## Backup MongoDB
echo "MongoDB $src_namespace port forward"
kubectl port-forward svc/rsmart-mongodb -n "$src_namespace" 27017 &
echo "Backup MongoDB"
echo "password: $src_mongodb_password" > config-mongodb.yaml
echo "uri: mongodb://admin:$src_mongodb_password@localhost:27017/?replicaSet=rs0&directConnection=true&authMechanism=SCRAM-SHA-1" >> config-mongodb.yaml
mongodump --config "./config-mongodb.yaml"
pkill -f "kubectl port-forward svc/rsmart-mongodb -n $src_namespace 27017"
## Get MongoDB pass from destination namespace
echo "Get MongoDB password from namespace: $dst_namespace"
dst_mongodb_password=$(kubectl exec -it svc/rsmart-mongodb -n "$dst_namespace" -c mongodb -- env | grep MONGODB_ROOT_PASSWORD | cut -d '=' -f 2)
echo "MongoDB $dst_namespace port forward"
kubectl port-forward svc/rsmart-mongodb -n "$dst_namespace" 27017 &
echo "Restaure MongoDB dump on $dst_namespace DB"
echo "password: $dst_mongodb_password" > config-mongodb.yaml
echo "uri: mongodb://admin:$dst_mongodb_password@localhost:27017/?replicaSet=rs0&directConnection=true&authMechanism=SCRAM-SHA-1" >> config-mongodb.yaml
mongorestore --config "./config-mongodb.yaml" --drop dump
pkill -f "kubectl port-forward svc/rsmart-mongodb -n $dst_namespace 27017"
echo "MongoDB script to anonymize DB"
CLEAN_PASS=$(echo "$dst_mongodb_password" | tr -cd '[:alnum:]')
kubectl exec -i svc/rsmart-mongodb -n "$dst_namespace" -- mongosh -u admin --password "$CLEAN_PASS" < $ANONYMIZE_SCRIPT
mdb_executed=true
elif [ "$mdb_option" == "n" ]; then
echo "Option MDB désactivée, aucune action à effectuer."
else
echo "Option MDB invalide. Utilisez 'y' ou 'n'."
exit 1
fi
}
# Function for tsdb option
function process_tsdb {
if [ "$tsdb_option" == "y" ]; then
echo "Traitement TSDB..."
echo "Backup TimescaleDB from $src_namespace"
kubectl exec -i svc/timescale -n "$src_namespace" -c timescaledb -- pg_dumpall --quote-all-identifiers --roles-only > roles.sql
kubectl exec -i svc/timescale -n "$src_namespace" -c timescaledb -- pg_dump --format=plain --quote-all-identifiers --no-owner -d measurement > measurement.sql
echo "Restaure TimescaleDB to $dst_namespace"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors < ./roles.sql
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "DROP SCHEMA IF EXISTS mtngeconnect CASCADE;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -c "DROP DATABASE measurement;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -c "CREATE DATABASE measurement TABLESPACE measurement OWNER quantumleap ENCODING 'UTF8';"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement < ./measurement.sql
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "ALTER SCHEMA mtngeconnect OWNER TO quantumleap ;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "${UPDATE_TABLE_OWNER}"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "ALTER TABLE public.md_ets_metadata OWNER TO quantumleap;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT ALL ON ALL TABLES IN SCHEMA mtngeconnect TO quantumleap;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA mtngeconnect TO quantumleap;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT SELECT ON ALL TABLES IN SCHEMA mtngeconnect TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "ALTER DEFAULT PRIVILEGES IN SCHEMA mtngeconnect GRANT SELECT ON TABLES TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA mtngeconnect TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT USAGE ON SCHEMA mtngeconnect TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT CONNECT ON DATABASE measurement TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT USAGE ON SCHEMA public TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT pg_monitor TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "GRANT SELECT ON ALL TABLES IN SCHEMA public TO quantumleapreader;"
kubectl exec -i svc/timescale -n "$dst_namespace" -c timescaledb -- psql -v --echo-errors -d measurement -c "ALTER TABLE schema_migrations OWNER TO quantumleap;"
tsdb_executed=true
elif [ "$tsdb_option" == "n" ]; then
echo "Option TSDB désactivée, aucune action à effectuer."
else
echo "Option TSDB invalide. Utilisez 'y' ou 'n'."
exit 1
fi
}
# call functions
scale_pods
process_mdb
process_tsdb
scale_pods
I have noticed that for mongodb, i still didn't verify for timescale. Thank you in advance for your answers.
Upvotes: 0
Views: 39