...
Note, this document will not NOT discuss the backup and restore for the application software itself (ie: all the software/dependencies you installed or symlinked into /opt) or the configuration for it. Use whatever conventional tool(s) you are comfortable with to backup/restore that.
This document will go over the simplest & most portable method (using the CQL COPY ... TO
/ COPY ... FROM
CQL commands), but be aware that other methods (ie: sstableloader
with nodetool snapshot
+ schema & token export/import or 3rd party ETL tools) are available too but have their own caveats. Refer to the Apache Cassandra and DataStax documentation (see ICX KB article "How to Backup & Restore Syndeia Cloud", https://docs.datastax.com/en/cassandra/3.0/cassandra/operations/migrating.html & https://web.archive.org/web/20161227010514/http://datascale.io/cloning-cassandra-clusters-fast-way/) for further details.
...
1. Copy and paste the below into a new file named syndeia_cloud-3.34_backup.bash
(set the password for syndeia_admin
on L4 (default is myPw
)):
...
Code Block | ||||||
---|---|---|---|---|---|---|
| ||||||
#!/bin/env bash cassandra_jg_host=${1:-localhost} version=${version:-3.4} syndeia_admin_pw=${syndeia_admin_pw:-myPw} SC_HOME=${SC_HOME:-/opt/icx/syndeia-cloud-current} keyspaces=(syndeia_cloud_auth syndeia_cloud_devops syndeia_cloud_graph syndeia_cloud_graph_config syndeia_cloud_store) tables=(syndeia_cloud_storeauth.snapshotsauto_key \/ syndeia_cloud_storeauth.metadataconfig \/ syndeia_cloud_storeauth.offsetstoremessages \/ syndeia_cloud_storeauth.auto_keymetadata \/ syndeia_cloud_storeauth.messagesoffsetstore \/ syndeia_cloud_storeauth.repositoriessnapshots \/ syndeia_cloud_store.repositories_by_uri \auth.users / syndeia_cloud_store.containers \ syndeia_cloud_store.containersauth.users_by_repositoryexternal_keyid \/ syndeia_cloud_storeauth.containersusers_by_external_key \/ syndeia_cloud_storeauth.containersusers_by_externaluser_idname \/ syndeia_cloud_storegraph.artifactsedgestore \/ syndeia_cloud_storegraph.artifactsedgestore_bylock_key \/ syndeia_cloud_store.artifacts_by_container_key \graph.graphindex / syndeia_cloud_graph.graphindex_lock_ / syndeia_cloud_storegraph.artifacts_by_external_key \janusgraph_ids / syndeia_cloud_storegraph.artifacts_by_external_id \system_properties / syndeia_cloud_storegraph.artifactssystem_byproperties_typelock_key \/ syndeia_cloud_storegraph.relationssystemlog \/ syndeia_cloud_store.relations_by_container_key \graph.txlog / syndeia_cloud_storegraph_config.configedgestore \/ syndeia_cloud_storegraph_config.relationsedgestore_bylock_key \/ syndeia_cloud_store.relations_by_external_id \graph_config.graphindex / syndeia_cloud_storegraph_config.relationsgraphindex_bylock_external_key \/ syndeia_cloud_graph_storeconfig.relations_by_source_key \janusgraph_ids / syndeia_cloud_storegraph_config.relations_by_target_key \system_properties / syndeia_cloud_store.repository_types \graph_config.system_properties_lock_ / syndeia_cloud_graph_storeconfig.container_typessystemlog \/ syndeia_cloud_store.container_types_by_repository_key \graph_config.txlog / syndeia_cloud_store.containerartifact_types_by_external_key \ / syndeia_cloud_store.containerartifact_types_by_external_id \/ syndeia_cloud_store.artifact_types \_by_external_key / syndeia_cloud_store.artifact_types_by_repository_key \/ syndeia_cloud_store.artifact_typesartifacts / syndeia_cloud_store.artifacts_by_externalcontainer_key \/ syndeia_cloud_store.artifactartifacts_types_by_external_id \/ syndeia_cloud_store.relation_types \artifacts_by_external_key / syndeia_cloud_store.relation_typesartifacts_by_repository_key \/ syndeia_cloud_store.relation_typesartifacts_by_externaltype_key \/ syndeia_cloud_store.relation_types_by_external_id \auto_key / syndeia_cloud_authstore.snapshotsconfig \/ syndeia_cloud_authstore.metadatacontainer_types \/ syndeia_cloud_auth.offsetstore \ syndeia_cloud_auth.messages \store.container_types_by_external_id / syndeia_cloud_auth.config \store.container_types_by_external_key / syndeia_cloud_auth.autostore.container_types_by_repository_key \/ syndeia_cloud_authstore.userscontainers \/ syndeia_cloud_authstore.userscontainers_by_external_id \/ syndeia_cloud_authstore.userscontainers_by_external_key \/ syndeia_cloud_authstore.userscontainers_by_user_name) LC_NUMERIC=C # TIMEFORMAT='{"real":%R,"user":%U,"sys":%S}' TIMEFORMAT='{"real":%R}' if [[ ! -d SC_backups ]]; then mkdir SC_backups fi; echo "$(date) - Dump schema for ${keyspaces}..." for keyspace in ${keyspaces[@]} do; dst_filename="SC_backups/${keyspace}_schema_backup.cql" all_keyspace_filenames="${all_keyspace_filenames} ${keyspace}_schema_backup.cql" echo "$(date) - repository_key / syndeia_cloud_store.messages / syndeia_cloud_store.metadata / syndeia_cloud_store.offsetstore / syndeia_cloud_store.relation_types / syndeia_cloud_store.relation_types_by_external_id / syndeia_cloud_store.relation_types_by_external_key / syndeia_cloud_store.relation_types_by_repository_key / syndeia_cloud_store.relations / syndeia_cloud_store.relations_by_container_key / syndeia_cloud_store.relations_by_external_id / syndeia_cloud_store.relations_by_external_key / syndeia_cloud_store.relations_by_key / syndeia_cloud_store.relations_by_source_key / syndeia_cloud_store.relations_by_target_key / syndeia_cloud_store.repositories / syndeia_cloud_store.repositories_by_uri / syndeia_cloud_store.repository_types / syndeia_cloud_store.snapshots) LC_NUMERIC=C # TIMEFORMAT='{"real":%R,"user":%U,"sys":%S}' TIMEFORMAT='{"real":%R}' if [[ ! -d SC_backups ]]; then mkdir SC_backups fi; echo "$(date) - Dump schema for ${keyspacekeyspaces}..." for keyspace in sudo cqlsh -u syndeia_admin -p ${syndeia_admin_pw} -e "DESCRIBE KEYSPACE ${keyspace};" "${cassandra_jg_host}" > ${dst_filename} > /dev/null if [[ $? -ne 0 ]]; then echo "$(date) x Error, could not export schema ${keyspace}" exit 1 ${keyspaces[@]} do; dst_filename="SC_backups/${keyspace}_schema_backup.cql" all_keyspace_filenames="${all_keyspace_filenames} ${keyspace}_schema_backup.cql" echo "$(date) - Dump schema for ${keyspace}..." sudo cqlsh -u syndeia_admin -p ${syndeia_admin_pw} -e "DESCRIBE KEYSPACE ${keyspace};" "${cassandra_jg_host}" > ${dst_filename} > /dev/null if [[ $? -ne 0 ]]; then echo "$(date) x Error, could not export schema ${keyspace}" exit 1 else echo "$(date) - Successfully exported schema ${keyspace}" fi; cat done; cat "${all_keyspace_filenames}" > SC_backups/syndeia-cloud-${version}_concatenated_schema_backup.cql > /dev/null echo "$(date) - Export data for tables..." for table in ${tables[@]} do; dst_filename="SC_backups/syndeia-cloud-${version}_backup_${table}.csv" # Generate MD5 hash for NULL values during each table export md5hash=$(date -Iseconds | md5sum | cut -f1 -d' ') echo "${md5hash}" > SC_backups/${table}_md5hash_for_NULL.txt echo "$(date) - Export data for ${table} to ${dst_filename} using md5hash=${md5hash} for NULL..." CQL_to_backup_table="COPY ${table} TO '${dst_filename}' WITH HEADER = TRUE AND NULL='${md5hash}'; " sudo cqlsh -u syndeia_admin -p ${syndeia_admin_pw} -e "${CQL_to_backup_table}" "${cassandra_jg_host}" > /dev/null if [[ $? -ne 0 ]]; then echo "$(date) x Error, could not export table ${table}" exit 2 else echo "$(date) - Successfully exported table ${table}" fi; done; |
...
Code Block | ||||||||
---|---|---|---|---|---|---|---|---|
| #!/bin/env bash
cassandra_jg_host=${1:-localhost}
version=${version:-3.4}
syndeia_admin_pw=${syndeia_admin_pw:-myPw}
SC_HOME=${SC_HOME:-/opt/icx/syndeia-cloud-current}
keyspaces=(syndeia_cloud_auth syndeia_cloud_devops syndeia_cloud_graph syndeia_cloud_graph_config syndeia_cloud_store)
tables=(syndeia_cloud_store.snapshots \
syndeia_cloud_store.metadata \
syndeia_cloud_store.offsetstore \
syndeia_cloud_store.auto_key \
syndeia_cloud_store.messages \
syndeia_cloud_store.repositories \
syndeia_cloud_store.repositories_by_uri \
syndeia_cloud_store.containers \
syndeia_cloud_store.containers_by_repository_key \
syndeia_cloud_store.containers_by_external_key \
syndeia_cloud_store.containers_by_external_id \
syndeia_cloud_store.artifacts \
syndeia_cloud_store.artifacts_by_key \
| |||||||
#!/bin/env bash cassandra_jg_host=${1:-localhost} version=${version:-3.4} syndeia_admin_pw=${syndeia_admin_pw:-myPw} SC_HOME=${SC_HOME:-/opt/icx/syndeia-cloud-current} keyspaces=(syndeia_cloud_auth syndeia_cloud_devops syndeia_cloud_graph syndeia_cloud_graph_config syndeia_cloud_store) tables=(syndeia_cloud_auth.auto_key / syndeia_cloud_auth.config / syndeia_cloud_auth.messages / syndeia_cloud_auth.metadata / syndeia_cloud_auth.offsetstore / syndeia_cloud_auth.snapshots / syndeia_cloud_auth.users / syndeia_cloud_auth.users_by_external_id / syndeia_cloud_auth.users_by_external_key / syndeia_cloud_auth.users_by_user_name / syndeia_cloud_graph.edgestore / syndeia_cloud_graph.edgestore_lock_ / syndeia_cloud_graph.graphindex / syndeia_cloud_graph.graphindex_lock_ / syndeia_cloud_graph.janusgraph_ids / syndeia_cloud_graph.system_properties / syndeia_cloud_graph.system_properties_lock_ / syndeia_cloud_graph.systemlog / syndeia_cloud_graph.txlog / syndeia_cloud_graph_config.edgestore / syndeia_cloud_graph_config.edgestore_lock_ / syndeia_cloud_graph_config.graphindex / syndeia_cloud_graph_config.graphindex_lock_ / syndeia_cloud_graph_config.janusgraph_ids / syndeia_cloud_graph_config.system_properties / syndeia_cloud_graph_config.system_properties_lock_ / syndeia_cloud_graph_config.systemlog / syndeia_cloud_graph_config.txlog / syndeia_cloud_store.artifact_types / syndeia_cloud_store.artifact_types_by_external_id / syndeia_cloud_store.artifact_types_by_external_key / syndeia_cloud_store.artifact_types_by_repository_key / syndeia_cloud_store.artifacts_by_container_key \ / syndeia_cloud_store.artifacts_by_externalcontainer_key \/ syndeia_cloud_store.artifacts_by_external_id \/ syndeia_cloud_store.artifacts_by_typeexternal_key \/ syndeia_cloud_store.relations \artifacts_by_key / syndeia_cloud_store.relationsartifacts_by_containertype_key \/ syndeia_cloud_store.configauto_key \/ syndeia_cloud_store.relations_by_key \config / syndeia_cloud_store.relations_by_external_id \container_types / syndeia_cloud_store.relationscontainer_types_by_external_keyid \/ syndeia_cloud_store.relationscontainer_types_by_sourceexternal_key \/ syndeia_cloud_store.relationscontainer_types_by_targetrepository_key \/ syndeia_cloud_store.repository_typescontainers \/ syndeia_cloud_store.container_types \containers_by_external_id / syndeia_cloud_store.container_typescontainers_by_repositoryexternal_key \/ syndeia_cloud_store.containercontainers_types_by_externalrepository_key \/ syndeia_cloud_store.container_types_by_external_id \messages / syndeia_cloud_store.artifact_typesmetadata \/ syndeia_cloud_store.artifact_types_by_repository_key \offsetstore / syndeia_cloud_store.artifactrelation_types_by_external_key \ / syndeia_cloud_store.artifactrelation_types_by_external_id \/ syndeia_cloud_store.relation_types \_by_external_key / syndeia_cloud_store.relation_types_by_repository_key \/ syndeia_cloud_store.relation_typesrelations / syndeia_cloud_store.relations_by_externalcontainer_key \/ syndeia_cloud_store.relation_typesrelations_by_external_id \/ syndeia_cloud_auth.snapshots \ syndeia_cloud_auth.metadata \ syndeia_cloud_auth.offsetstore \store.relations_by_external_key / syndeia_cloud_auth.messages \store.relations_by_key / syndeia_cloud_auth.config \store.relations_by_source_key / syndeia_cloud_auth.autostore.relations_by_target_key \/ syndeia_cloud_authstore.usersrepositories \/ syndeia_cloud_authstore.usersrepositories_by_external_iduri \/ syndeia_cloud_authstore.users_by_external_key \repository_types / syndeia_cloud_auth.users_by_user_namestore.snapshots) LC_NUMERIC=C # TIMEFORMAT='{"real":%R,"user":%U,"sys":%S}' TIMEFORMAT='{"real":%R}' echo "$(date) - Restore schema for ${keyspaces}..." src_filename="SC_backups/syndeia-cloud-${version}_concatenated_schema_backup.cql" sudo cqlsh -u syndeia_admin -p ${syndeia_admin_pw} -e "SOURCE '${src_filename}';" "${cassandra_jg_host}" if [[ $? -ne 0 ]]; then echo "$(date) x Error, could not import concatenated schema ${src_filename}" exit 1 else echo "$(date) - Successfully imported concatenated schema ${src_filename}" fi; echo "$(date) - Import data for tables..." for table in ${tables[@]} do; src_filename="SC_backups/syndeia-cloud-${version}_backup_${table}.csv" # Read MD5 hash for NULL values during each table export md5hash=$(cat SC_backups/${table}_md5hash_for_NULL.txt) echo "$(date) - Import data for ${table} from ${dst_filename}..." CQL_to_restore_table="COPY ${table} FROM '${src_filename}' WITH HEADER = TRUE AND WITH NULL='${md5hash}'; " sudo cqlsh -u syndeia_admin -p ${syndeia_admin_pw} -e "${CQL_to_restore_table}" "${cassandra_jg_host}" if [[ $? -ne 0 ]]; then echo "$(date) x Error, could not import table ${table}" exit 2 else echo "$(date) - Successfully imported table ${table}" fi; done; echo "$(date) - Done!" |
...