From fd7787bf18bb4bca63e71b823c93e2e36fc2e34b Mon Sep 17 00:00:00 2001 From: Ludwig Krispenz Date: Wed, 31 Jan 2018 09:22:10 +0100 Subject: [PATCH 3/3] move cldb - export changelog with backend --- ldap/admin/src/scripts/db2ldif.in | 4 +- ldap/admin/src/scripts/ldif2db.in | 4 +- ldap/servers/plugins/replication/cl5_api.c | 59 +++++++++++++++++++++++++++ ldap/servers/plugins/replication/cl5_api.h | 3 ++ ldap/servers/plugins/replication/repl5_init.c | 2 + ldap/servers/slapd/back-ldbm/ldif2ldbm.c | 8 ++++ ldap/servers/slapd/main.c | 20 ++++++++- ldap/servers/slapd/pblock.c | 35 ++++++++++++++++ ldap/servers/slapd/pblock_v3.h | 1 + ldap/servers/slapd/plugin.c | 4 ++ ldap/servers/slapd/slap.h | 4 ++ ldap/servers/slapd/slapi-plugin.h | 3 ++ 12 files changed, 144 insertions(+), 3 deletions(-) diff --git a/ldap/admin/src/scripts/db2ldif.in b/ldap/admin/src/scripts/db2ldif.in index 95d2754..6e3af53 100755 --- a/ldap/admin/src/scripts/db2ldif.in +++ b/ldap/admin/src/scripts/db2ldif.in @@ -25,6 +25,7 @@ usage() echo " -x - Suffix to exclude" echo " -a outputfile - Name of the exported LDIF file" echo " -r - Include replication data" + echo " -R - Include changelog data" echo " -E - Decrypt attributes" echo " -u - Do not export the nsUniqueId attribute" echo " -U - Do not wrap long lines" @@ -99,7 +100,7 @@ then exit 1 fi -while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qc:" flag +while getopts "hZ:vd:D:ENa:rs:x:CSut:n:UmMo1qRc:" flag do case $flag in h) usage @@ -125,6 +126,7 @@ do M) args=$args" -M";; 1) args=$args" -1";; q) args=$args" -q";; + R) args=$args" -R";; c) cwd=$OPTARG;; ?) usage exit 1;; diff --git a/ldap/admin/src/scripts/ldif2db.in b/ldap/admin/src/scripts/ldif2db.in index 20c7d46..9097d16 100755 --- a/ldap/admin/src/scripts/ldif2db.in +++ b/ldap/admin/src/scripts/ldif2db.in @@ -30,6 +30,7 @@ usage() echo " -G name - Namespace id for name based uniqueid (-g deterministic)" echo " -O - Do not index the attributes" echo " -E - Encrypt attributes" + echo " -R - Import changelog data" echo " -q - Quiet mode - suppresses output" echo " -v - Display version" echo " -h - Display usage" @@ -53,7 +54,7 @@ handleopts() return 0 } -while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:Eq" flag +while getopts "Z:vhd:i:g:G:n:s:x:NOCc:St:D:ERq" flag do case $flag in h) usage @@ -70,6 +71,7 @@ do t) args=$args" -t \"$OPTARG\"";; D) args=$args" -D \"$OPTARG\"";; E) args=$args" -E";; + R) args=$args" -R";; v) args=$args" -v";; N) args=$args" -N";; C) args=$args" -C";; diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c index 7320316..4c5a4e5 100644 --- a/ldap/servers/plugins/replication/cl5_api.c +++ b/ldap/servers/plugins/replication/cl5_api.c @@ -4972,6 +4972,65 @@ bail: return rc; } +static char * +_cl5LdifFileName(char *instance_ldif) +{ + char *cl_ldif = NULL; + char *p =strstr(instance_ldif, ".ldif"); + + if (p) { + *p = '\0'; + cl_ldif = slapi_ch_smprintf("%s_cl.ldif", instance_ldif); + } else { + cl_ldif = slapi_ch_smprintf("%s_cl", instance_ldif); + } + + return cl_ldif; + +} + +int +cl5Import(Slapi_PBlock *pb) +{ + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cl5Export - Importing changelog\n"); + return 0; +} + +int +cl5Export(Slapi_PBlock *pb) +{ + char *instance_name; + char *instance_ldif; + char *instance_cl_ldif; + Slapi_Backend *be; + Object *repl_obj; + Replica *replica = NULL; + int rc; + + + + slapi_pblock_get(pb, SLAPI_BACKEND_INSTANCE_NAME, &instance_name); + slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &instance_ldif); + slapi_pblock_get(pb, SLAPI_BACKEND, &be); + if (repl_obj = replica_get_replica_from_dn(slapi_be_getsuffix(be, 0))) { + replica = (Replica *)object_get_data(repl_obj); + } + if (replica == NULL) { + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cl5Export - No replica defined for instance %s\n", instance_name); + return 0; + } + + instance_cl_ldif = _cl5LdifFileName(instance_ldif); + slapi_log_err(SLAPI_LOG_ERR, repl_plugin_name_cl, + "cl5Export - Exporting changelog for instance %s to file %s\n", + instance_name, instance_cl_ldif); + rc = cl5ExportLDIF(instance_cl_ldif, replica); + + return rc; +} + /* * Clean the in memory RUV, at shutdown we will write the update to the db */ diff --git a/ldap/servers/plugins/replication/cl5_api.h b/ldap/servers/plugins/replication/cl5_api.h index ec22571..0a58098 100644 --- a/ldap/servers/plugins/replication/cl5_api.h +++ b/ldap/servers/plugins/replication/cl5_api.h @@ -420,6 +420,9 @@ int cl5WriteRUV(void); Parameters: none Return: TRUE */ +int cl5Export(Slapi_PBlock *pb); +int cl5Import(Slapi_PBlock *pb); + int cl5DeleteRUV(void); void cl5CleanRUV(ReplicaId rid, Replica *replica); void cl5NotifyCleanup(int rid); diff --git a/ldap/servers/plugins/replication/repl5_init.c b/ldap/servers/plugins/replication/repl5_init.c index 6afd0fe..6b118ac 100644 --- a/ldap/servers/plugins/replication/repl5_init.c +++ b/ldap/servers/plugins/replication/repl5_init.c @@ -407,6 +407,8 @@ multimaster_betxn_bepostop_init(Slapi_PBlock *pb) if (slapi_pblock_set(pb, SLAPI_PLUGIN_VERSION, SLAPI_PLUGIN_VERSION_01) || slapi_pblock_set(pb, SLAPI_PLUGIN_DESCRIPTION, (void *)&multimasterbepostopdesc) || + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN, (void *)cl5Export) || + slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_IMPORT_FN, (void *)cl5Import) || slapi_pblock_set(pb, SLAPI_PLUGIN_BE_POST_BACKUP_FN, (void *)cl5DeleteRUV)) { slapi_log_err(SLAPI_LOG_PLUGIN, repl_plugin_name, "multimaster_betxn_bepostop_init - Failed\n"); rc = -1; diff --git a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c index 16b87ee..261bb03 100644 --- a/ldap/servers/slapd/back-ldbm/ldif2ldbm.c +++ b/ldap/servers/slapd/back-ldbm/ldif2ldbm.c @@ -1115,6 +1115,7 @@ ldbm_back_ldbm2ldif(Slapi_PBlock *pb) int decrypt = 0; int32_t dump_replica = 0; int dump_uniqueid = 1; + int dump_changelog = 0; int fd = STDOUT_FILENO; IDList *idl = NULL; /* optimization for -s include lists */ int cnt = 0, lastcnt = 0; @@ -1231,6 +1232,7 @@ ldbm_back_ldbm2ldif(Slapi_PBlock *pb) slapi_pblock_get(pb, SLAPI_DB2LDIF_FILE, &fname); slapi_pblock_get(pb, SLAPI_DB2LDIF_PRINTKEY, &printkey); slapi_pblock_get(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &dump_uniqueid); + slapi_pblock_get(pb, SLAPI_LDIF_CHANGELOG, &dump_changelog); /* tsk, overloading printkey. shame on me. */ ok_index = !(printkey & EXPORT_ID2ENTRY_ONLY); @@ -1629,6 +1631,12 @@ ldbm_back_ldbm2ldif(Slapi_PBlock *pb) "export %s: Processed %d entries (100%%).\n", inst->inst_name, cnt); } + if (run_from_cmdline && dump_changelog) { + return_value = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_EXPORT_FN); + slapi_log_err(SLAPI_LOG_INFO, "ldbm_back_ldbm2ldif", + "export changelog for %s.\n", inst->inst_name); + } + bye: if (idl) { idl_free(&idl); diff --git a/ldap/servers/slapd/main.c b/ldap/servers/slapd/main.c index 521d314..99042be 100644 --- a/ldap/servers/slapd/main.c +++ b/ldap/servers/slapd/main.c @@ -85,6 +85,7 @@ struct main_config int ldif2db_noattrindexes; char **db2index_attrs; int ldif_printkey; + int ldif_include_changelog; char *archive_name; int db2ldif_dump_replica; int db2ldif_dump_uniqueid; @@ -642,6 +643,7 @@ main(int argc, char **argv) /* Set a number of defaults */ mcfg.slapd_exemode = SLAPD_EXEMODE_UNKNOWN; mcfg.ldif_printkey = EXPORT_PRINTKEY | EXPORT_APPENDMODE; + mcfg.ldif_include_changelog = 0; mcfg.db2ldif_dump_uniqueid = 1; mcfg.ldif2db_generate_uniqueid = SLAPI_UNIQUEID_GENERATE_TIME_BASED; mcfg.ldif2db_removedupvals = 1; @@ -1276,7 +1278,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg) * */ - char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1q"; + char *opts_db2ldif = "vd:D:ENa:rs:x:CSut:n:UmMo1qR"; struct opt_ext long_options_db2ldif[] = { {"version", ArgNone, 'v'}, {"debug", ArgRequired, 'd'}, @@ -1296,6 +1298,7 @@ process_command_line(int argc, char **argv, struct main_config *mcfg) {"multipleOutputFile", ArgNone, 'M'}, {"noVersionNum", ArgNone, '1'}, {"quiet", ArgNone, 'q'}, + {"includechangelog", ArgNone, 'R'}, {0, 0, 0}}; char *opts_ldif2db = "vd:i:g:G:n:s:x:NOCc:St:D:Eq"; @@ -1692,6 +1695,20 @@ process_command_line(int argc, char **argv, struct main_config *mcfg) break; + case 'R': /* db2ldif and ldif2db only */ + if (mcfg->slapd_exemode != SLAPD_EXEMODE_DB2LDIF && + mcfg->slapd_exemode != SLAPD_EXEMODE_LDIF2DB) { + usage(mcfg->myname, mcfg->extraname, mcfg->slapd_exemode); + exit(1); + } + + /* + * import/export should handle changelog. + */ + mcfg->ldif_include_changelog = 1; + + break; + case 'C': if (mcfg->slapd_exemode == SLAPD_EXEMODE_LDIF2DB) { /* used to mean "Cool new import" (which is now @@ -2266,6 +2283,7 @@ slapd_exemode_db2ldif(int argc, char **argv, struct main_config *mcfg) slapi_pblock_set(pb, SLAPI_BACKEND_INSTANCE_NAME, *instp); slapi_pblock_set_ldif_dump_replica(pb, mcfg->db2ldif_dump_replica); slapi_pblock_set(pb, SLAPI_DB2LDIF_DUMP_UNIQUEID, &(mcfg->db2ldif_dump_uniqueid)); + slapi_pblock_set(pb, SLAPI_LDIF_CHANGELOG, &(mcfg->ldif_include_changelog)); int32_t task_flags = SLAPI_TASK_RUNNING_FROM_COMMANDLINE; slapi_pblock_set(pb, SLAPI_TASK_FLAGS, &task_flags); int32_t is_running = 0; diff --git a/ldap/servers/slapd/pblock.c b/ldap/servers/slapd/pblock.c index 4514c3c..6482a61 100644 --- a/ldap/servers/slapd/pblock.c +++ b/ldap/servers/slapd/pblock.c @@ -1276,6 +1276,18 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) } (*(IFP *)value) = pblock->pb_plugin->plg_bepostbackup; break; + case SLAPI_PLUGIN_BE_POST_EXPORT_FN: + if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { + return (-1); + } + (*(IFP *)value) = pblock->pb_plugin->plg_bepostexport; + break; + case SLAPI_PLUGIN_BE_POST_IMPORT_FN: + if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { + return (-1); + } + (*(IFP *)value) = pblock->pb_plugin->plg_bepostimport; + break; /* internal preoperation plugin */ case SLAPI_PLUGIN_INTERNAL_PRE_MODIFY_FN: @@ -2090,6 +2102,13 @@ slapi_pblock_get(Slapi_PBlock *pblock, int arg, void *value) (*(int *)value) = 0; } break; + case SLAPI_LDIF_CHANGELOG: + if (pblock->pb_task != NULL) { + (*(int *)value) = pblock->pb_task->ldif_include_changelog; + } else { + (*(int *)value) = 0; + } + break; /* dbverify */ case SLAPI_DBVERIFY_DBDIR: if (pblock->pb_task != NULL) { @@ -3171,6 +3190,18 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) } pblock->pb_plugin->plg_bepostbackup = (IFP)value; break; + case SLAPI_PLUGIN_BE_POST_EXPORT_FN: + if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { + return (-1); + } + pblock->pb_plugin->plg_bepostexport = (IFP)value; + break; + case SLAPI_PLUGIN_BE_POST_IMPORT_FN: + if (pblock->pb_plugin->plg_type != SLAPI_PLUGIN_BEPOSTOPERATION) { + return (-1); + } + pblock->pb_plugin->plg_bepostimport = (IFP)value; + break; /* internal preoperation plugin */ case SLAPI_PLUGIN_INTERNAL_PRE_MODIFY_FN: @@ -3880,6 +3911,10 @@ slapi_pblock_set(Slapi_PBlock *pblock, int arg, void *value) _pblock_assert_pb_task(pblock); pblock->pb_task->import_state = *((int *)value); break; + case SLAPI_LDIF_CHANGELOG: + _pblock_assert_pb_task(pblock); + pblock->pb_task->ldif_include_changelog = *((int *)value); + break; case SLAPI_LDIF2DB_ENCRYPT: case SLAPI_DB2LDIF_DECRYPT: diff --git a/ldap/servers/slapd/pblock_v3.h b/ldap/servers/slapd/pblock_v3.h index 7ec2f37..ffcdf7c 100644 --- a/ldap/servers/slapd/pblock_v3.h +++ b/ldap/servers/slapd/pblock_v3.h @@ -60,6 +60,7 @@ typedef struct _slapi_pblock_task char *ldif_namespaceid; /* used for name based uniqueid generation */ int ldif_dump_replica; int ldif_dump_uniqueid; /* dump uniqueid during db2ldif */ + int ldif_include_changelog; /* include changelog for import/export */ int ldif_generate_uniqueid; /* generate uniqueid during db2ldif */ int ldif_encrypt; /* used to enable encrypt/decrypt on import and export */ int seq_type; diff --git a/ldap/servers/slapd/plugin.c b/ldap/servers/slapd/plugin.c index b175f74..5b02193 100644 --- a/ldap/servers/slapd/plugin.c +++ b/ldap/servers/slapd/plugin.c @@ -388,6 +388,8 @@ plugin_call_plugins(Slapi_PBlock *pb, int whichfunction) case SLAPI_PLUGIN_BE_POST_DELETE_FN: case SLAPI_PLUGIN_BE_POST_OPEN_FN: case SLAPI_PLUGIN_BE_POST_BACKUP_FN: + case SLAPI_PLUGIN_BE_POST_EXPORT_FN: + case SLAPI_PLUGIN_BE_POST_IMPORT_FN: plugin_list_number = PLUGIN_LIST_BEPOSTOPERATION; do_op = 1; /* always allow backend callbacks (even during startup) */ break; @@ -3625,6 +3627,8 @@ plugin_invoke_plugin_pb(struct slapdplugin *plugin, int operation, Slapi_PBlock operation == SLAPI_PLUGIN_BE_PRE_CLOSE_FN || operation == SLAPI_PLUGIN_BE_POST_OPEN_FN || operation == SLAPI_PLUGIN_BE_PRE_BACKUP_FN || + operation == SLAPI_PLUGIN_BE_POST_EXPORT_FN || + operation == SLAPI_PLUGIN_BE_POST_IMPORT_FN || operation == SLAPI_PLUGIN_BE_POST_BACKUP_FN) return PR_TRUE; diff --git a/ldap/servers/slapd/slap.h b/ldap/servers/slapd/slap.h index 70605f9..1e59b0b 100644 --- a/ldap/servers/slapd/slap.h +++ b/ldap/servers/slapd/slap.h @@ -1175,6 +1175,8 @@ struct slapdplugin IFP plg_un_bepost_delete; /* delete */ IFP plg_un_bepost_open; /* open */ IFP plg_un_bepost_backup; /* backup */ + IFP plg_un_bepost_import; /* import */ + IFP plg_un_bepost_export; /* export */ } plg_un_bepost; #define plg_bepostmodify plg_un.plg_un_bepost.plg_un_bepost_modify #define plg_bepostmodrdn plg_un.plg_un_bepost.plg_un_bepost_modrdn @@ -1182,6 +1184,8 @@ struct slapdplugin #define plg_bepostdelete plg_un.plg_un_bepost.plg_un_bepost_delete #define plg_bepostopen plg_un.plg_un_bepost.plg_un_bepost_open #define plg_bepostbackup plg_un.plg_un_bepost.plg_un_bepost_backup +#define plg_bepostimport plg_un.plg_un_bepost.plg_un_bepost_import +#define plg_bepostexport plg_un.plg_un_bepost.plg_un_bepost_export /* internal pre-operation plugin structure */ struct plg_un_internal_pre_operation diff --git a/ldap/servers/slapd/slapi-plugin.h b/ldap/servers/slapd/slapi-plugin.h index 0559af2..d82c79a 100644 --- a/ldap/servers/slapd/slapi-plugin.h +++ b/ldap/servers/slapd/slapi-plugin.h @@ -7109,6 +7109,8 @@ typedef struct slapi_plugindesc #define SLAPI_PLUGIN_BE_POST_DELETE_FN 553 #define SLAPI_PLUGIN_BE_POST_OPEN_FN 554 #define SLAPI_PLUGIN_BE_POST_BACKUP_FN 555 +#define SLAPI_PLUGIN_BE_POST_EXPORT_FN 556 +#define SLAPI_PLUGIN_BE_POST_IMPORT_FN 557 /* postoperation plugin to the backend - just before transaction commit */ #define SLAPI_PLUGIN_BE_TXN_POST_ADD_FN 560 @@ -7403,6 +7405,7 @@ typedef struct slapi_plugindesc #define SLAPI_DB2LDIF_FILE 184 /* dump uniqueid */ #define SLAPI_DB2LDIF_DUMP_UNIQUEID 176 +#define SLAPI_LDIF_CHANGELOG 1761 #define SLAPI_DB2LDIF_SERVER_RUNNING 197 /* db2ldif/ldif2db/bak2db/db2bak arguments */ -- 2.9.5