001/* 002 * Copyright 2016-2020 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2016-2020 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2016-2020 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.transformations; 037 038 039 040import java.io.File; 041import java.io.FileOutputStream; 042import java.io.InputStream; 043import java.io.OutputStream; 044import java.util.ArrayList; 045import java.util.EnumSet; 046import java.util.Iterator; 047import java.util.LinkedHashMap; 048import java.util.List; 049import java.util.Set; 050import java.util.TreeMap; 051import java.util.concurrent.atomic.AtomicLong; 052import java.util.zip.GZIPOutputStream; 053 054import com.unboundid.ldap.sdk.Attribute; 055import com.unboundid.ldap.sdk.ChangeType; 056import com.unboundid.ldap.sdk.DN; 057import com.unboundid.ldap.sdk.Entry; 058import com.unboundid.ldap.sdk.LDAPException; 059import com.unboundid.ldap.sdk.ResultCode; 060import com.unboundid.ldap.sdk.Version; 061import com.unboundid.ldap.sdk.schema.Schema; 062import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 063import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator; 064import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator; 065import com.unboundid.ldif.LDIFException; 066import com.unboundid.ldif.LDIFReader; 067import com.unboundid.ldif.LDIFReaderChangeRecordTranslator; 068import com.unboundid.ldif.LDIFReaderEntryTranslator; 069import com.unboundid.ldif.LDIFRecord; 070import com.unboundid.util.ByteStringBuffer; 071import com.unboundid.util.CommandLineTool; 072import com.unboundid.util.Debug; 073import com.unboundid.util.ObjectPair; 074import com.unboundid.util.PassphraseEncryptedOutputStream; 075import com.unboundid.util.StaticUtils; 076import com.unboundid.util.ThreadSafety; 077import com.unboundid.util.ThreadSafetyLevel; 078import com.unboundid.util.args.ArgumentException; 079import com.unboundid.util.args.ArgumentParser; 080import com.unboundid.util.args.BooleanArgument; 081import com.unboundid.util.args.DNArgument; 082import com.unboundid.util.args.FileArgument; 083import com.unboundid.util.args.FilterArgument; 084import com.unboundid.util.args.IntegerArgument; 085import com.unboundid.util.args.ScopeArgument; 086import com.unboundid.util.args.StringArgument; 087 088import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*; 089 090 091 092/** 093 * This class provides a command-line tool that can be used to apply a number of 094 * transformations to an LDIF file. The transformations that can be applied 095 * include: 096 * <UL> 097 * <LI> 098 * It can scramble the values of a specified set of attributes in a manner 099 * that attempts to preserve the syntax and consistently scrambles the same 100 * value to the same representation. 101 * </LI> 102 * <LI> 103 * It can strip a specified set of attributes out of entries. 104 * </LI> 105 * <LI> 106 * It can redact the values of a specified set of attributes, to indicate 107 * that the values are there but providing no information about what their 108 * values are. 109 * </LI> 110 * <LI> 111 * It can replace the values of a specified attribute with a given set of 112 * values. 113 * </LI> 114 * <LI> 115 * It can add an attribute with a given set of values to any entry that does 116 * not contain that attribute. 117 * </LI> 118 * <LI> 119 * It can replace the values of a specified attribute with a value that 120 * contains a sequentially-incrementing counter. 121 * </LI> 122 * <LI> 123 * It can strip entries matching a given base DN, scope, and filter out of 124 * the LDIF file. 125 * </LI> 126 * <LI> 127 * It can perform DN mapping, so that entries that exist below one base DN 128 * are moved below a different base DN. 129 * </LI> 130 * <LI> 131 * It can perform attribute mapping, to replace uses of one attribute name 132 * with another. 133 * </LI> 134 * </UL> 135 */ 136@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 137public final class TransformLDIF 138 extends CommandLineTool 139 implements LDIFReaderEntryTranslator 140{ 141 /** 142 * The maximum length of any message to write to standard output or standard 143 * error. 144 */ 145 private static final int MAX_OUTPUT_LINE_LENGTH = 146 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 147 148 149 150 // The arguments for use by this program. 151 private BooleanArgument addToExistingValues = null; 152 private BooleanArgument appendToTargetLDIF = null; 153 private BooleanArgument compressTarget = null; 154 private BooleanArgument encryptTarget = null; 155 private BooleanArgument excludeRecordsWithoutChangeType = null; 156 private BooleanArgument excludeNonMatchingEntries = null; 157 private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null; 158 private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null; 159 private BooleanArgument hideRedactedValueCount = null; 160 private BooleanArgument processDNs = null; 161 private BooleanArgument sourceCompressed = null; 162 private BooleanArgument sourceContainsChangeRecords = null; 163 private BooleanArgument sourceFromStandardInput = null; 164 private BooleanArgument targetToStandardOutput = null; 165 private DNArgument addAttributeBaseDN = null; 166 private DNArgument excludeEntryBaseDN = null; 167 private DNArgument flattenBaseDN = null; 168 private DNArgument moveSubtreeFrom = null; 169 private DNArgument moveSubtreeTo = null; 170 private FileArgument encryptionPassphraseFile = null; 171 private FileArgument schemaPath = null; 172 private FileArgument sourceLDIF = null; 173 private FileArgument targetLDIF = null; 174 private FilterArgument addAttributeFilter = null; 175 private FilterArgument excludeEntryFilter = null; 176 private FilterArgument flattenExcludeFilter = null; 177 private IntegerArgument initialSequentialValue = null; 178 private IntegerArgument numThreads = null; 179 private IntegerArgument randomSeed = null; 180 private IntegerArgument sequentialValueIncrement = null; 181 private IntegerArgument wrapColumn = null; 182 private ScopeArgument addAttributeScope = null; 183 private ScopeArgument excludeEntryScope = null; 184 private StringArgument addAttributeName = null; 185 private StringArgument addAttributeValue = null; 186 private StringArgument excludeAttribute = null; 187 private StringArgument excludeChangeType = null; 188 private StringArgument redactAttribute = null; 189 private StringArgument renameAttributeFrom = null; 190 private StringArgument renameAttributeTo = null; 191 private StringArgument replaceValuesAttribute = null; 192 private StringArgument replacementValue = null; 193 private StringArgument scrambleAttribute = null; 194 private StringArgument scrambleJSONField = null; 195 private StringArgument sequentialAttribute = null; 196 private StringArgument textAfterSequentialValue = null; 197 private StringArgument textBeforeSequentialValue = null; 198 199 // A set of thread-local byte stream buffers that will be used to construct 200 // the LDIF representations of records. 201 private final ThreadLocal<ByteStringBuffer> byteStringBuffers = 202 new ThreadLocal<>(); 203 204 205 206 /** 207 * Invokes this tool with the provided set of arguments. 208 * 209 * @param args The command-line arguments provided to this program. 210 */ 211 public static void main(final String... args) 212 { 213 final ResultCode resultCode = main(System.out, System.err, args); 214 if (resultCode != ResultCode.SUCCESS) 215 { 216 System.exit(resultCode.intValue()); 217 } 218 } 219 220 221 222 /** 223 * Invokes this tool with the provided set of arguments. 224 * 225 * @param out The output stream to use for standard output. It may be 226 * {@code null} if standard output should be suppressed. 227 * @param err The output stream to use for standard error. It may be 228 * {@code null} if standard error should be suppressed. 229 * @param args The command-line arguments provided to this program. 230 * 231 * @return A result code indicating whether processing completed 232 * successfully. 233 */ 234 public static ResultCode main(final OutputStream out, final OutputStream err, 235 final String... args) 236 { 237 final TransformLDIF tool = new TransformLDIF(out, err); 238 return tool.runTool(args); 239 } 240 241 242 243 /** 244 * Creates a new instance of this tool with the provided information. 245 * 246 * @param out The output stream to use for standard output. It may be 247 * {@code null} if standard output should be suppressed. 248 * @param err The output stream to use for standard error. It may be 249 * {@code null} if standard error should be suppressed. 250 */ 251 public TransformLDIF(final OutputStream out, final OutputStream err) 252 { 253 super(out, err); 254 } 255 256 257 258 /** 259 * {@inheritDoc} 260 */ 261 @Override() 262 public String getToolName() 263 { 264 return "transform-ldif"; 265 } 266 267 268 269 /** 270 * {@inheritDoc} 271 */ 272 @Override() 273 public String getToolDescription() 274 { 275 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get(); 276 } 277 278 279 280 /** 281 * {@inheritDoc} 282 */ 283 @Override() 284 public String getToolVersion() 285 { 286 return Version.NUMERIC_VERSION_STRING; 287 } 288 289 290 291 /** 292 * {@inheritDoc} 293 */ 294 @Override() 295 public boolean supportsInteractiveMode() 296 { 297 return true; 298 } 299 300 301 302 /** 303 * {@inheritDoc} 304 */ 305 @Override() 306 public boolean defaultsToInteractiveMode() 307 { 308 return true; 309 } 310 311 312 313 /** 314 * {@inheritDoc} 315 */ 316 @Override() 317 public boolean supportsPropertiesFile() 318 { 319 return true; 320 } 321 322 323 324 /** 325 * {@inheritDoc} 326 */ 327 @Override() 328 public void addToolArguments(final ArgumentParser parser) 329 throws ArgumentException 330 { 331 // Add arguments pertaining to the source and target LDIF files. 332 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null, 333 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true, 334 false); 335 sourceLDIF.addLongIdentifier("inputLDIF", true); 336 sourceLDIF.addLongIdentifier("source-ldif", true); 337 sourceLDIF.addLongIdentifier("input-ldif", true); 338 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 339 parser.addArgument(sourceLDIF); 340 341 sourceFromStandardInput = new BooleanArgument(null, 342 "sourceFromStandardInput", 1, 343 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get()); 344 sourceFromStandardInput.addLongIdentifier("source-from-standard-input", 345 true); 346 sourceFromStandardInput.setArgumentGroupName( 347 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 348 parser.addArgument(sourceFromStandardInput); 349 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput); 350 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput); 351 352 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null, 353 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true, 354 false); 355 targetLDIF.addLongIdentifier("outputLDIF", true); 356 targetLDIF.addLongIdentifier("target-ldif", true); 357 targetLDIF.addLongIdentifier("output-ldif", true); 358 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 359 parser.addArgument(targetLDIF); 360 361 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput", 362 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get()); 363 targetToStandardOutput.addLongIdentifier("target-to-standard-output", true); 364 targetToStandardOutput.setArgumentGroupName( 365 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 366 parser.addArgument(targetToStandardOutput); 367 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput); 368 369 sourceContainsChangeRecords = new BooleanArgument(null, 370 "sourceContainsChangeRecords", 371 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get()); 372 sourceContainsChangeRecords.addLongIdentifier( 373 "source-contains-change-records", true); 374 sourceContainsChangeRecords.setArgumentGroupName( 375 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 376 parser.addArgument(sourceContainsChangeRecords); 377 378 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF", 379 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get()); 380 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true); 381 appendToTargetLDIF.setArgumentGroupName( 382 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 383 parser.addArgument(appendToTargetLDIF); 384 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF); 385 386 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null, 387 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE); 388 wrapColumn.addLongIdentifier("wrap-column", true); 389 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 390 parser.addArgument(wrapColumn); 391 392 sourceCompressed = new BooleanArgument('C', "sourceCompressed", 393 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get()); 394 sourceCompressed.addLongIdentifier("inputCompressed", true); 395 sourceCompressed.addLongIdentifier("source-compressed", true); 396 sourceCompressed.addLongIdentifier("input-compressed", true); 397 sourceCompressed.setArgumentGroupName( 398 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 399 parser.addArgument(sourceCompressed); 400 401 compressTarget = new BooleanArgument('c', "compressTarget", 402 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get()); 403 compressTarget.addLongIdentifier("compressOutput", true); 404 compressTarget.addLongIdentifier("compress", true); 405 compressTarget.addLongIdentifier("compress-target", true); 406 compressTarget.addLongIdentifier("compress-output", true); 407 compressTarget.setArgumentGroupName( 408 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 409 parser.addArgument(compressTarget); 410 411 encryptTarget = new BooleanArgument(null, "encryptTarget", 412 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get()); 413 encryptTarget.addLongIdentifier("encryptOutput", true); 414 encryptTarget.addLongIdentifier("encrypt", true); 415 encryptTarget.addLongIdentifier("encrypt-target", true); 416 encryptTarget.addLongIdentifier("encrypt-output", true); 417 encryptTarget.setArgumentGroupName( 418 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 419 parser.addArgument(encryptTarget); 420 421 encryptionPassphraseFile = new FileArgument(null, 422 "encryptionPassphraseFile", false, 1, null, 423 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true, 424 true, false); 425 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 426 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 427 true); 428 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 429 true); 430 encryptionPassphraseFile.setArgumentGroupName( 431 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 432 parser.addArgument(encryptionPassphraseFile); 433 434 435 // Add arguments pertaining to attribute scrambling. 436 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0, 437 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 438 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get()); 439 scrambleAttribute.addLongIdentifier("attributeName", true); 440 scrambleAttribute.addLongIdentifier("scramble-attribute", true); 441 scrambleAttribute.addLongIdentifier("attribute-name", true); 442 scrambleAttribute.setArgumentGroupName( 443 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 444 parser.addArgument(scrambleAttribute); 445 446 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0, 447 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(), 448 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get( 449 scrambleAttribute.getIdentifierString())); 450 scrambleJSONField.addLongIdentifier("scramble-json-field", true); 451 scrambleJSONField.setArgumentGroupName( 452 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 453 parser.addArgument(scrambleJSONField); 454 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute); 455 456 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null, 457 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get()); 458 randomSeed.addLongIdentifier("random-seed", true); 459 randomSeed.setArgumentGroupName( 460 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 461 parser.addArgument(randomSeed); 462 463 464 // Add arguments pertaining to replacing attribute values with a generated 465 // value using a sequential counter. 466 sequentialAttribute = new StringArgument('S', "sequentialAttribute", 467 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 468 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get( 469 sourceContainsChangeRecords.getIdentifierString())); 470 sequentialAttribute.addLongIdentifier("sequentialAttributeName", true); 471 sequentialAttribute.addLongIdentifier("sequential-attribute", true); 472 sequentialAttribute.addLongIdentifier("sequential-attribute-name", true); 473 sequentialAttribute.setArgumentGroupName( 474 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 475 parser.addArgument(sequentialAttribute); 476 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 477 sequentialAttribute); 478 479 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue", 480 false, 1, null, 481 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get( 482 sequentialAttribute.getIdentifierString())); 483 initialSequentialValue.addLongIdentifier("initial-sequential-value", true); 484 initialSequentialValue.setArgumentGroupName( 485 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 486 parser.addArgument(initialSequentialValue); 487 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute); 488 489 sequentialValueIncrement = new IntegerArgument(null, 490 "sequentialValueIncrement", false, 1, null, 491 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get( 492 sequentialAttribute.getIdentifierString())); 493 sequentialValueIncrement.addLongIdentifier("sequential-value-increment", 494 true); 495 sequentialValueIncrement.setArgumentGroupName( 496 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 497 parser.addArgument(sequentialValueIncrement); 498 parser.addDependentArgumentSet(sequentialValueIncrement, 499 sequentialAttribute); 500 501 textBeforeSequentialValue = new StringArgument(null, 502 "textBeforeSequentialValue", false, 1, null, 503 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get( 504 sequentialAttribute.getIdentifierString())); 505 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value", 506 true); 507 textBeforeSequentialValue.setArgumentGroupName( 508 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 509 parser.addArgument(textBeforeSequentialValue); 510 parser.addDependentArgumentSet(textBeforeSequentialValue, 511 sequentialAttribute); 512 513 textAfterSequentialValue = new StringArgument(null, 514 "textAfterSequentialValue", false, 1, null, 515 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get( 516 sequentialAttribute.getIdentifierString())); 517 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value", 518 true); 519 textAfterSequentialValue.setArgumentGroupName( 520 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 521 parser.addArgument(textAfterSequentialValue); 522 parser.addDependentArgumentSet(textAfterSequentialValue, 523 sequentialAttribute); 524 525 526 // Add arguments pertaining to attribute value replacement. 527 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute", 528 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 529 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get( 530 sourceContainsChangeRecords.getIdentifierString())); 531 replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true); 532 replaceValuesAttribute.setArgumentGroupName( 533 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 534 parser.addArgument(replaceValuesAttribute); 535 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 536 replaceValuesAttribute); 537 538 replacementValue = new StringArgument(null, "replacementValue", false, 0, 539 null, 540 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get( 541 replaceValuesAttribute.getIdentifierString())); 542 replacementValue.addLongIdentifier("replacement-value", true); 543 replacementValue.setArgumentGroupName( 544 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 545 parser.addArgument(replacementValue); 546 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue); 547 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute); 548 549 550 // Add arguments pertaining to adding missing attributes. 551 addAttributeName = new StringArgument(null, "addAttributeName", false, 1, 552 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 553 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get( 554 "--addAttributeValue", 555 sourceContainsChangeRecords.getIdentifierString())); 556 addAttributeName.addLongIdentifier("add-attribute-name", true); 557 addAttributeName.setArgumentGroupName( 558 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 559 parser.addArgument(addAttributeName); 560 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 561 addAttributeName); 562 563 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0, 564 null, 565 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get( 566 addAttributeName.getIdentifierString())); 567 addAttributeValue.addLongIdentifier("add-attribute-value", true); 568 addAttributeValue.setArgumentGroupName( 569 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 570 parser.addArgument(addAttributeValue); 571 parser.addDependentArgumentSet(addAttributeName, addAttributeValue); 572 parser.addDependentArgumentSet(addAttributeValue, addAttributeName); 573 574 addToExistingValues = new BooleanArgument(null, "addToExistingValues", 575 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get( 576 addAttributeName.getIdentifierString(), 577 addAttributeValue.getIdentifierString())); 578 addToExistingValues.addLongIdentifier("add-to-existing-values", true); 579 addToExistingValues.setArgumentGroupName( 580 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 581 parser.addArgument(addToExistingValues); 582 parser.addDependentArgumentSet(addToExistingValues, addAttributeName); 583 584 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1, 585 null, 586 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get( 587 addAttributeName.getIdentifierString())); 588 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true); 589 addAttributeBaseDN.setArgumentGroupName( 590 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 591 parser.addArgument(addAttributeBaseDN); 592 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName); 593 594 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false, 595 null, 596 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get( 597 addAttributeBaseDN.getIdentifierString(), 598 addAttributeName.getIdentifierString())); 599 addAttributeScope.addLongIdentifier("add-attribute-scope", true); 600 addAttributeScope.setArgumentGroupName( 601 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 602 parser.addArgument(addAttributeScope); 603 parser.addDependentArgumentSet(addAttributeScope, addAttributeName); 604 605 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false, 606 1, null, 607 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get( 608 addAttributeName.getIdentifierString())); 609 addAttributeFilter.addLongIdentifier("add-attribute-filter", true); 610 addAttributeFilter.setArgumentGroupName( 611 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 612 parser.addArgument(addAttributeFilter); 613 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName); 614 615 616 // Add arguments pertaining to renaming attributes. 617 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom", 618 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 619 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get()); 620 renameAttributeFrom.addLongIdentifier("rename-attribute-from", true); 621 renameAttributeFrom.setArgumentGroupName( 622 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 623 parser.addArgument(renameAttributeFrom); 624 625 renameAttributeTo = new StringArgument(null, "renameAttributeTo", 626 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 627 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get( 628 renameAttributeFrom.getIdentifierString())); 629 renameAttributeTo.addLongIdentifier("rename-attribute-to", true); 630 renameAttributeTo.setArgumentGroupName( 631 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 632 parser.addArgument(renameAttributeTo); 633 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo); 634 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom); 635 636 637 // Add arguments pertaining to flattening subtrees. 638 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null, 639 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get()); 640 flattenBaseDN.addLongIdentifier("flatten-base-dn", true); 641 flattenBaseDN.setArgumentGroupName( 642 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 643 parser.addArgument(flattenBaseDN); 644 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 645 flattenBaseDN); 646 647 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null, 648 "flattenAddOmittedRDNAttributesToEntry", 1, 649 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get()); 650 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier( 651 "flatten-add-omitted-rdn-attributes-to-entry", true); 652 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName( 653 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 654 parser.addArgument(flattenAddOmittedRDNAttributesToEntry); 655 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry, 656 flattenBaseDN); 657 658 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null, 659 "flattenAddOmittedRDNAttributesToRDN", 1, 660 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get()); 661 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier( 662 "flatten-add-omitted-rdn-attributes-to-rdn", true); 663 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName( 664 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 665 parser.addArgument(flattenAddOmittedRDNAttributesToRDN); 666 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN, 667 flattenBaseDN); 668 669 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter", 670 false, 1, null, 671 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get()); 672 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true); 673 flattenExcludeFilter.setArgumentGroupName( 674 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 675 parser.addArgument(flattenExcludeFilter); 676 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN); 677 678 679 // Add arguments pertaining to moving subtrees. 680 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null, 681 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get()); 682 moveSubtreeFrom.addLongIdentifier("move-subtree-from", true); 683 moveSubtreeFrom.setArgumentGroupName( 684 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 685 parser.addArgument(moveSubtreeFrom); 686 687 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null, 688 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get( 689 moveSubtreeFrom.getIdentifierString())); 690 moveSubtreeTo.addLongIdentifier("move-subtree-to", true); 691 moveSubtreeTo.setArgumentGroupName( 692 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 693 parser.addArgument(moveSubtreeTo); 694 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo); 695 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom); 696 697 698 // Add arguments pertaining to redacting attribute values. 699 redactAttribute = new StringArgument(null, "redactAttribute", false, 0, 700 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 701 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get()); 702 redactAttribute.addLongIdentifier("redact-attribute", true); 703 redactAttribute.setArgumentGroupName( 704 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 705 parser.addArgument(redactAttribute); 706 707 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount", 708 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get()); 709 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count", 710 true); 711 hideRedactedValueCount.setArgumentGroupName( 712 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 713 parser.addArgument(hideRedactedValueCount); 714 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute); 715 716 717 // Add arguments pertaining to excluding attributes and entries. 718 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0, 719 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 720 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get()); 721 excludeAttribute.addLongIdentifier("suppressAttribute", true); 722 excludeAttribute.addLongIdentifier("exclude-attribute", true); 723 excludeAttribute.addLongIdentifier("suppress-attribute", true); 724 excludeAttribute.setArgumentGroupName( 725 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 726 parser.addArgument(excludeAttribute); 727 728 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1, 729 null, 730 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get( 731 sourceContainsChangeRecords.getIdentifierString())); 732 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true); 733 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true); 734 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true); 735 excludeEntryBaseDN.setArgumentGroupName( 736 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 737 parser.addArgument(excludeEntryBaseDN); 738 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 739 excludeEntryBaseDN); 740 741 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false, 742 null, 743 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get( 744 sourceContainsChangeRecords.getIdentifierString())); 745 excludeEntryScope.addLongIdentifier("suppressEntryScope", true); 746 excludeEntryScope.addLongIdentifier("exclude-entry-scope", true); 747 excludeEntryScope.addLongIdentifier("suppress-entry-scope", true); 748 excludeEntryScope.setArgumentGroupName( 749 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 750 parser.addArgument(excludeEntryScope); 751 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 752 excludeEntryScope); 753 754 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false, 755 1, null, 756 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get( 757 sourceContainsChangeRecords.getIdentifierString())); 758 excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true); 759 excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true); 760 excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true); 761 excludeEntryFilter.setArgumentGroupName( 762 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 763 parser.addArgument(excludeEntryFilter); 764 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 765 excludeEntryFilter); 766 767 excludeNonMatchingEntries = new BooleanArgument(null, 768 "excludeNonMatchingEntries", 769 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get()); 770 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries", 771 true); 772 excludeNonMatchingEntries.setArgumentGroupName( 773 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 774 parser.addArgument(excludeNonMatchingEntries); 775 parser.addDependentArgumentSet(excludeNonMatchingEntries, 776 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 777 778 779 // Add arguments for excluding records based on their change types. 780 excludeChangeType = new StringArgument(null, "excludeChangeType", 781 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(), 782 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(), 783 StaticUtils.setOf("add", "delete", "modify", "moddn")); 784 excludeChangeType.addLongIdentifier("exclude-change-type", true); 785 excludeChangeType.addLongIdentifier("exclude-changetype", true); 786 excludeChangeType.setArgumentGroupName( 787 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 788 parser.addArgument(excludeChangeType); 789 790 791 // Add arguments for excluding records that don't have a change type. 792 excludeRecordsWithoutChangeType = new BooleanArgument(null, 793 "excludeRecordsWithoutChangeType", 1, 794 INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get()); 795 excludeRecordsWithoutChangeType.addLongIdentifier( 796 "exclude-records-without-change-type", true); 797 excludeRecordsWithoutChangeType.addLongIdentifier( 798 "exclude-records-without-changetype", true); 799 excludeRecordsWithoutChangeType.setArgumentGroupName( 800 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 801 parser.addArgument(excludeRecordsWithoutChangeType); 802 803 804 // Add the remaining arguments. 805 schemaPath = new FileArgument(null, "schemaPath", false, 0, null, 806 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(), 807 true, true, false, false); 808 schemaPath.addLongIdentifier("schemaFile", true); 809 schemaPath.addLongIdentifier("schemaDirectory", true); 810 schemaPath.addLongIdentifier("schema-path", true); 811 schemaPath.addLongIdentifier("schema-file", true); 812 schemaPath.addLongIdentifier("schema-directory", true); 813 parser.addArgument(schemaPath); 814 815 numThreads = new IntegerArgument('t', "numThreads", false, 1, null, 816 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE, 817 1); 818 numThreads.addLongIdentifier("num-threads", true); 819 parser.addArgument(numThreads); 820 821 processDNs = new BooleanArgument('d', "processDNs", 822 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get()); 823 processDNs.addLongIdentifier("process-dns", true); 824 parser.addArgument(processDNs); 825 826 827 // Ensure that at least one kind of transformation was requested. 828 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute, 829 replaceValuesAttribute, addAttributeName, renameAttributeFrom, 830 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute, 831 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter, 832 excludeChangeType, excludeRecordsWithoutChangeType); 833 } 834 835 836 837 /** 838 * {@inheritDoc} 839 */ 840 @Override() 841 public void doExtendedArgumentValidation() 842 throws ArgumentException 843 { 844 // Ideally, exactly one of the targetLDIF and targetToStandardOutput 845 // arguments should always be provided. But in order to preserve backward 846 // compatibility with a legacy scramble-ldif tool, we will allow both to be 847 // omitted if either --scrambleAttribute or --sequentialArgument is 848 // provided. In that case, the path of the output file will be the path of 849 // the first input file with ".scrambled" appended to it. 850 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent())) 851 { 852 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent())) 853 { 854 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get( 855 targetLDIF.getIdentifierString(), 856 targetToStandardOutput.getIdentifierString())); 857 } 858 } 859 860 861 // Make sure that the --renameAttributeFrom and --renameAttributeTo 862 // arguments were provided an equal number of times. 863 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences(); 864 final int renameToOccurrences = renameAttributeTo.getNumOccurrences(); 865 if (renameFromOccurrences != renameToOccurrences) 866 { 867 throw new ArgumentException( 868 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 869 renameAttributeFrom.getIdentifierString(), 870 renameAttributeTo.getIdentifierString())); 871 } 872 873 874 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were 875 // provided an equal number of times. 876 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences(); 877 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences(); 878 if (moveFromOccurrences != moveToOccurrences) 879 { 880 throw new ArgumentException( 881 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 882 moveSubtreeFrom.getIdentifierString(), 883 moveSubtreeTo.getIdentifierString())); 884 } 885 } 886 887 888 889 /** 890 * {@inheritDoc} 891 */ 892 @Override() 893 public ResultCode doToolProcessing() 894 { 895 final Schema schema; 896 try 897 { 898 schema = getSchema(); 899 } 900 catch (final LDAPException le) 901 { 902 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage()); 903 return le.getResultCode(); 904 } 905 906 907 // If an encryption passphrase file is provided, then get the passphrase 908 // from it. 909 String encryptionPassphrase = null; 910 if (encryptionPassphraseFile.isPresent()) 911 { 912 try 913 { 914 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 915 encryptionPassphraseFile.getValue()); 916 } 917 catch (final LDAPException e) 918 { 919 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage()); 920 return e.getResultCode(); 921 } 922 } 923 924 925 // Create the translators to use to apply the transformations. 926 final ArrayList<LDIFReaderEntryTranslator> entryTranslators = 927 new ArrayList<>(10); 928 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators = 929 new ArrayList<>(10); 930 931 final AtomicLong excludedEntryCount = new AtomicLong(0L); 932 createTranslators(entryTranslators, changeRecordTranslators, 933 schema, excludedEntryCount); 934 935 final AggregateLDIFReaderEntryTranslator entryTranslator = 936 new AggregateLDIFReaderEntryTranslator(entryTranslators); 937 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator = 938 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators); 939 940 941 // Determine the path to the target file to be written. 942 final File targetFile; 943 if (targetLDIF.isPresent()) 944 { 945 targetFile = targetLDIF.getValue(); 946 } 947 else if (targetToStandardOutput.isPresent()) 948 { 949 targetFile = null; 950 } 951 else 952 { 953 targetFile = 954 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled"); 955 } 956 957 958 // Create the LDIF reader. 959 final LDIFReader ldifReader; 960 try 961 { 962 final InputStream inputStream; 963 if (sourceLDIF.isPresent()) 964 { 965 final ObjectPair<InputStream,String> p = 966 ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(), 967 encryptionPassphrase, getOut(), getErr()); 968 inputStream = p.getFirst(); 969 if ((encryptionPassphrase == null) && (p.getSecond() != null)) 970 { 971 encryptionPassphrase = p.getSecond(); 972 } 973 } 974 else 975 { 976 inputStream = System.in; 977 } 978 979 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), 980 entryTranslator, changeRecordTranslator); 981 if (schema != null) 982 { 983 ldifReader.setSchema(schema); 984 } 985 } 986 catch (final Exception e) 987 { 988 Debug.debugException(e); 989 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 990 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get( 991 StaticUtils.getExceptionMessage(e))); 992 return ResultCode.LOCAL_ERROR; 993 } 994 995 996 ResultCode resultCode = ResultCode.SUCCESS; 997 OutputStream outputStream = null; 998processingBlock: 999 try 1000 { 1001 // Create the output stream to use to write the transformed data. 1002 try 1003 { 1004 if (targetFile == null) 1005 { 1006 outputStream = getOut(); 1007 } 1008 else 1009 { 1010 outputStream = 1011 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent()); 1012 } 1013 1014 if (encryptTarget.isPresent()) 1015 { 1016 if (encryptionPassphrase == null) 1017 { 1018 encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase( 1019 false, true, getOut(), getErr()); 1020 } 1021 1022 outputStream = new PassphraseEncryptedOutputStream( 1023 encryptionPassphrase, outputStream); 1024 } 1025 1026 if (compressTarget.isPresent()) 1027 { 1028 outputStream = new GZIPOutputStream(outputStream); 1029 } 1030 } 1031 catch (final Exception e) 1032 { 1033 Debug.debugException(e); 1034 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1035 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get( 1036 targetFile.getAbsolutePath(), 1037 StaticUtils.getExceptionMessage(e))); 1038 resultCode = ResultCode.LOCAL_ERROR; 1039 break processingBlock; 1040 } 1041 1042 1043 // Read the source data one record at a time. The transformations will 1044 // automatically be applied by the LDIF reader's translators, and even if 1045 // there are multiple reader threads, we're guaranteed to get the results 1046 // in the right order. 1047 long entriesWritten = 0L; 1048 while (true) 1049 { 1050 final LDIFRecord ldifRecord; 1051 try 1052 { 1053 ldifRecord = ldifReader.readLDIFRecord(); 1054 } 1055 catch (final LDIFException le) 1056 { 1057 Debug.debugException(le); 1058 if (le.mayContinueReading()) 1059 { 1060 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1061 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get( 1062 StaticUtils.getExceptionMessage(le))); 1063 if (resultCode == ResultCode.SUCCESS) 1064 { 1065 resultCode = ResultCode.PARAM_ERROR; 1066 } 1067 continue; 1068 } 1069 else 1070 { 1071 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1072 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get( 1073 StaticUtils.getExceptionMessage(le))); 1074 if (resultCode == ResultCode.SUCCESS) 1075 { 1076 resultCode = ResultCode.PARAM_ERROR; 1077 } 1078 break processingBlock; 1079 } 1080 } 1081 catch (final Exception e) 1082 { 1083 Debug.debugException(e); 1084 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1085 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get( 1086 StaticUtils.getExceptionMessage(e))); 1087 resultCode = ResultCode.LOCAL_ERROR; 1088 break processingBlock; 1089 } 1090 1091 1092 // If the LDIF record is null, then we've run out of records so we're 1093 // done. 1094 if (ldifRecord == null) 1095 { 1096 break; 1097 } 1098 1099 1100 // Write the record to the output stream. 1101 try 1102 { 1103 if (ldifRecord instanceof PreEncodedLDIFEntry) 1104 { 1105 outputStream.write( 1106 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes()); 1107 } 1108 else 1109 { 1110 final ByteStringBuffer buffer = getBuffer(); 1111 if (wrapColumn.isPresent()) 1112 { 1113 ldifRecord.toLDIF(buffer, wrapColumn.getValue()); 1114 } 1115 else 1116 { 1117 ldifRecord.toLDIF(buffer, 0); 1118 } 1119 buffer.append(StaticUtils.EOL_BYTES); 1120 buffer.write(outputStream); 1121 } 1122 } 1123 catch (final Exception e) 1124 { 1125 Debug.debugException(e); 1126 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1127 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(), 1128 StaticUtils.getExceptionMessage(e))); 1129 resultCode = ResultCode.LOCAL_ERROR; 1130 break processingBlock; 1131 } 1132 1133 1134 // If we've written a multiple of 1000 entries, print a progress 1135 // message. 1136 entriesWritten++; 1137 if ((! targetToStandardOutput.isPresent()) && 1138 ((entriesWritten % 1000L) == 0)) 1139 { 1140 final long numExcluded = excludedEntryCount.get(); 1141 if (numExcluded > 0L) 1142 { 1143 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1144 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get( 1145 entriesWritten, numExcluded)); 1146 } 1147 else 1148 { 1149 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1150 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get( 1151 entriesWritten)); 1152 } 1153 } 1154 } 1155 1156 1157 if (! targetToStandardOutput.isPresent()) 1158 { 1159 final long numExcluded = excludedEntryCount.get(); 1160 if (numExcluded > 0L) 1161 { 1162 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1163 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten, 1164 numExcluded)); 1165 } 1166 else 1167 { 1168 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1169 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten)); 1170 } 1171 } 1172 } 1173 finally 1174 { 1175 if (outputStream != null) 1176 { 1177 try 1178 { 1179 outputStream.close(); 1180 } 1181 catch (final Exception e) 1182 { 1183 Debug.debugException(e); 1184 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1185 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get( 1186 targetFile.getAbsolutePath(), 1187 StaticUtils.getExceptionMessage(e))); 1188 if (resultCode == ResultCode.SUCCESS) 1189 { 1190 resultCode = ResultCode.LOCAL_ERROR; 1191 } 1192 } 1193 } 1194 1195 try 1196 { 1197 ldifReader.close(); 1198 } 1199 catch (final Exception e) 1200 { 1201 Debug.debugException(e); 1202 // We can ignore this. 1203 } 1204 } 1205 1206 1207 return resultCode; 1208 } 1209 1210 1211 1212 /** 1213 * Retrieves the schema that should be used for processing. 1214 * 1215 * @return The schema that was created. 1216 * 1217 * @throws LDAPException If a problem is encountered while retrieving the 1218 * schema. 1219 */ 1220 private Schema getSchema() 1221 throws LDAPException 1222 { 1223 // If any schema paths were specified, then load the schema only from those 1224 // paths. 1225 if (schemaPath.isPresent()) 1226 { 1227 final ArrayList<File> schemaFiles = new ArrayList<>(10); 1228 for (final File path : schemaPath.getValues()) 1229 { 1230 if (path.isFile()) 1231 { 1232 schemaFiles.add(path); 1233 } 1234 else 1235 { 1236 final TreeMap<String,File> fileMap = new TreeMap<>(); 1237 for (final File schemaDirFile : path.listFiles()) 1238 { 1239 final String name = schemaDirFile.getName(); 1240 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif")) 1241 { 1242 fileMap.put(name, schemaDirFile); 1243 } 1244 } 1245 schemaFiles.addAll(fileMap.values()); 1246 } 1247 } 1248 1249 if (schemaFiles.isEmpty()) 1250 { 1251 throw new LDAPException(ResultCode.PARAM_ERROR, 1252 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get( 1253 schemaPath.getIdentifierString())); 1254 } 1255 else 1256 { 1257 try 1258 { 1259 return Schema.getSchema(schemaFiles); 1260 } 1261 catch (final Exception e) 1262 { 1263 Debug.debugException(e); 1264 throw new LDAPException(ResultCode.LOCAL_ERROR, 1265 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get( 1266 StaticUtils.getExceptionMessage(e))); 1267 } 1268 } 1269 } 1270 else 1271 { 1272 // If the INSTANCE_ROOT environment variable is set and it refers to a 1273 // directory that has a config/schema subdirectory that has one or more 1274 // schema files in it, then read the schema from that directory. 1275 try 1276 { 1277 final String instanceRootStr = 1278 StaticUtils.getEnvironmentVariable("INSTANCE_ROOT"); 1279 if (instanceRootStr != null) 1280 { 1281 final File instanceRoot = new File(instanceRootStr); 1282 final File configDir = new File(instanceRoot, "config"); 1283 final File schemaDir = new File(configDir, "schema"); 1284 if (schemaDir.exists()) 1285 { 1286 final TreeMap<String,File> fileMap = new TreeMap<>(); 1287 for (final File schemaDirFile : schemaDir.listFiles()) 1288 { 1289 final String name = schemaDirFile.getName(); 1290 if (schemaDirFile.isFile() && 1291 name.toLowerCase().endsWith(".ldif")) 1292 { 1293 fileMap.put(name, schemaDirFile); 1294 } 1295 } 1296 1297 if (! fileMap.isEmpty()) 1298 { 1299 return Schema.getSchema(new ArrayList<>(fileMap.values())); 1300 } 1301 } 1302 } 1303 } 1304 catch (final Exception e) 1305 { 1306 Debug.debugException(e); 1307 } 1308 } 1309 1310 1311 // If we've gotten here, then just return null and the tool will try to use 1312 // the default standard schema. 1313 return null; 1314 } 1315 1316 1317 1318 /** 1319 * Creates the entry and change record translators that will be used to 1320 * perform the transformations. 1321 * 1322 * @param entryTranslators A list to which all created entry 1323 * translators should be written. 1324 * @param changeRecordTranslators A list to which all created change record 1325 * translators should be written. 1326 * @param schema The schema to use when processing. 1327 * @param excludedEntryCount A counter used to keep track of the number 1328 * of entries that have been excluded from 1329 * the result set. 1330 */ 1331 private void createTranslators( 1332 final List<LDIFReaderEntryTranslator> entryTranslators, 1333 final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators, 1334 final Schema schema, final AtomicLong excludedEntryCount) 1335 { 1336 if (scrambleAttribute.isPresent()) 1337 { 1338 final Long seed; 1339 if (randomSeed.isPresent()) 1340 { 1341 seed = randomSeed.getValue().longValue(); 1342 } 1343 else 1344 { 1345 seed = null; 1346 } 1347 1348 final ScrambleAttributeTransformation t = 1349 new ScrambleAttributeTransformation(schema, seed, 1350 processDNs.isPresent(), scrambleAttribute.getValues(), 1351 scrambleJSONField.getValues()); 1352 entryTranslators.add(t); 1353 changeRecordTranslators.add(t); 1354 } 1355 1356 if (sequentialAttribute.isPresent()) 1357 { 1358 final long initialValue; 1359 if (initialSequentialValue.isPresent()) 1360 { 1361 initialValue = initialSequentialValue.getValue().longValue(); 1362 } 1363 else 1364 { 1365 initialValue = 0L; 1366 } 1367 1368 final long incrementAmount; 1369 if (sequentialValueIncrement.isPresent()) 1370 { 1371 incrementAmount = sequentialValueIncrement.getValue().longValue(); 1372 } 1373 else 1374 { 1375 incrementAmount = 1L; 1376 } 1377 1378 for (final String attrName : sequentialAttribute.getValues()) 1379 { 1380 1381 1382 final ReplaceWithCounterTransformation t = 1383 new ReplaceWithCounterTransformation(schema, attrName, 1384 initialValue, incrementAmount, 1385 textBeforeSequentialValue.getValue(), 1386 textAfterSequentialValue.getValue(), processDNs.isPresent()); 1387 entryTranslators.add(t); 1388 } 1389 } 1390 1391 if (replaceValuesAttribute.isPresent()) 1392 { 1393 final ReplaceAttributeTransformation t = 1394 new ReplaceAttributeTransformation(schema, 1395 replaceValuesAttribute.getValue(), 1396 replacementValue.getValues()); 1397 entryTranslators.add(t); 1398 } 1399 1400 if (addAttributeName.isPresent()) 1401 { 1402 final AddAttributeTransformation t = new AddAttributeTransformation( 1403 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(), 1404 addAttributeFilter.getValue(), 1405 new Attribute(addAttributeName.getValue(), schema, 1406 addAttributeValue.getValues()), 1407 (! addToExistingValues.isPresent())); 1408 entryTranslators.add(t); 1409 } 1410 1411 if (renameAttributeFrom.isPresent()) 1412 { 1413 final Iterator<String> renameFromIterator = 1414 renameAttributeFrom.getValues().iterator(); 1415 final Iterator<String> renameToIterator = 1416 renameAttributeTo.getValues().iterator(); 1417 while (renameFromIterator.hasNext()) 1418 { 1419 final RenameAttributeTransformation t = 1420 new RenameAttributeTransformation(schema, 1421 renameFromIterator.next(), renameToIterator.next(), 1422 processDNs.isPresent()); 1423 entryTranslators.add(t); 1424 changeRecordTranslators.add(t); 1425 } 1426 } 1427 1428 if (flattenBaseDN.isPresent()) 1429 { 1430 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation( 1431 schema, flattenBaseDN.getValue(), 1432 flattenAddOmittedRDNAttributesToEntry.isPresent(), 1433 flattenAddOmittedRDNAttributesToRDN.isPresent(), 1434 flattenExcludeFilter.getValue()); 1435 entryTranslators.add(t); 1436 } 1437 1438 if (moveSubtreeFrom.isPresent()) 1439 { 1440 final Iterator<DN> moveFromIterator = 1441 moveSubtreeFrom.getValues().iterator(); 1442 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator(); 1443 while (moveFromIterator.hasNext()) 1444 { 1445 final MoveSubtreeTransformation t = 1446 new MoveSubtreeTransformation(moveFromIterator.next(), 1447 moveToIterator.next()); 1448 entryTranslators.add(t); 1449 changeRecordTranslators.add(t); 1450 } 1451 } 1452 1453 if (redactAttribute.isPresent()) 1454 { 1455 final RedactAttributeTransformation t = new RedactAttributeTransformation( 1456 schema, processDNs.isPresent(), 1457 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues()); 1458 entryTranslators.add(t); 1459 changeRecordTranslators.add(t); 1460 } 1461 1462 if (excludeAttribute.isPresent()) 1463 { 1464 final ExcludeAttributeTransformation t = 1465 new ExcludeAttributeTransformation(schema, 1466 excludeAttribute.getValues()); 1467 entryTranslators.add(t); 1468 changeRecordTranslators.add(t); 1469 } 1470 1471 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() || 1472 excludeEntryFilter.isPresent()) 1473 { 1474 final ExcludeEntryTransformation t = new ExcludeEntryTransformation( 1475 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(), 1476 excludeEntryFilter.getValue(), 1477 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount); 1478 entryTranslators.add(t); 1479 } 1480 1481 if (excludeChangeType.isPresent()) 1482 { 1483 final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class); 1484 for (final String changeTypeName : excludeChangeType.getValues()) 1485 { 1486 changeTypes.add(ChangeType.forName(changeTypeName)); 1487 } 1488 1489 changeRecordTranslators.add( 1490 new ExcludeChangeTypeTransformation(changeTypes)); 1491 } 1492 1493 if (excludeRecordsWithoutChangeType.isPresent()) 1494 { 1495 entryTranslators.add(new ExcludeAllEntriesTransformation()); 1496 } 1497 1498 entryTranslators.add(this); 1499 } 1500 1501 1502 1503 /** 1504 * {@inheritDoc} 1505 */ 1506 @Override() 1507 public LinkedHashMap<String[],String> getExampleUsages() 1508 { 1509 final LinkedHashMap<String[],String> examples = 1510 new LinkedHashMap<>(StaticUtils.computeMapCapacity(4)); 1511 1512 examples.put( 1513 new String[] 1514 { 1515 "--sourceLDIF", "input.ldif", 1516 "--targetLDIF", "scrambled.ldif", 1517 "--scrambleAttribute", "givenName", 1518 "--scrambleAttribute", "sn", 1519 "--scrambleAttribute", "cn", 1520 "--numThreads", "10", 1521 "--schemaPath", "/ds/config/schema", 1522 "--processDNs" 1523 }, 1524 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get()); 1525 1526 examples.put( 1527 new String[] 1528 { 1529 "--sourceLDIF", "input.ldif", 1530 "--targetLDIF", "sequential.ldif", 1531 "--sequentialAttribute", "uid", 1532 "--initialSequentialValue", "1", 1533 "--sequentialValueIncrement", "1", 1534 "--textBeforeSequentialValue", "user.", 1535 "--numThreads", "10", 1536 "--schemaPath", "/ds/config/schema", 1537 "--processDNs" 1538 }, 1539 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get()); 1540 1541 examples.put( 1542 new String[] 1543 { 1544 "--sourceLDIF", "input.ldif", 1545 "--targetLDIF", "added-organization.ldif", 1546 "--addAttributeName", "o", 1547 "--addAttributeValue", "Example Corp.", 1548 "--addAttributeFilter", "(objectClass=person)", 1549 "--numThreads", "10", 1550 "--schemaPath", "/ds/config/schema" 1551 }, 1552 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get()); 1553 1554 examples.put( 1555 new String[] 1556 { 1557 "--sourceLDIF", "input.ldif", 1558 "--targetLDIF", "rebased.ldif", 1559 "--moveSubtreeFrom", "o=example.com", 1560 "--moveSubtreeTo", "dc=example,dc=com", 1561 "--numThreads", "10", 1562 "--schemaPath", "/ds/config/schema" 1563 }, 1564 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get()); 1565 1566 return examples; 1567 } 1568 1569 1570 1571 /** 1572 * {@inheritDoc} 1573 */ 1574 @Override() 1575 public Entry translate(final Entry original, final long firstLineNumber) 1576 throws LDIFException 1577 { 1578 final ByteStringBuffer buffer = getBuffer(); 1579 if (wrapColumn.isPresent()) 1580 { 1581 original.toLDIF(buffer, wrapColumn.getValue()); 1582 } 1583 else 1584 { 1585 original.toLDIF(buffer, 0); 1586 } 1587 buffer.append(StaticUtils.EOL_BYTES); 1588 1589 return new PreEncodedLDIFEntry(original, buffer.toByteArray()); 1590 } 1591 1592 1593 1594 /** 1595 * Retrieves a byte string buffer that can be used to perform LDIF encoding. 1596 * 1597 * @return A byte string buffer that can be used to perform LDIF encoding. 1598 */ 1599 private ByteStringBuffer getBuffer() 1600 { 1601 ByteStringBuffer buffer = byteStringBuffers.get(); 1602 if (buffer == null) 1603 { 1604 buffer = new ByteStringBuffer(); 1605 byteStringBuffers.set(buffer); 1606 } 1607 else 1608 { 1609 buffer.clear(); 1610 } 1611 1612 return buffer; 1613 } 1614}