001/* 002 * Copyright 2013-2020 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2013-2020 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2013-2020 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.examples; 037 038 039 040import java.io.OutputStream; 041import java.util.Collections; 042import java.util.LinkedHashMap; 043import java.util.List; 044import java.util.Map; 045import java.util.TreeMap; 046import java.util.concurrent.atomic.AtomicLong; 047 048import com.unboundid.asn1.ASN1OctetString; 049import com.unboundid.ldap.sdk.Attribute; 050import com.unboundid.ldap.sdk.DN; 051import com.unboundid.ldap.sdk.Filter; 052import com.unboundid.ldap.sdk.LDAPConnectionOptions; 053import com.unboundid.ldap.sdk.LDAPConnectionPool; 054import com.unboundid.ldap.sdk.LDAPException; 055import com.unboundid.ldap.sdk.LDAPSearchException; 056import com.unboundid.ldap.sdk.ResultCode; 057import com.unboundid.ldap.sdk.SearchRequest; 058import com.unboundid.ldap.sdk.SearchResult; 059import com.unboundid.ldap.sdk.SearchResultEntry; 060import com.unboundid.ldap.sdk.SearchResultReference; 061import com.unboundid.ldap.sdk.SearchResultListener; 062import com.unboundid.ldap.sdk.SearchScope; 063import com.unboundid.ldap.sdk.Version; 064import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl; 065import com.unboundid.util.Debug; 066import com.unboundid.util.LDAPCommandLineTool; 067import com.unboundid.util.StaticUtils; 068import com.unboundid.util.ThreadSafety; 069import com.unboundid.util.ThreadSafetyLevel; 070import com.unboundid.util.args.ArgumentException; 071import com.unboundid.util.args.ArgumentParser; 072import com.unboundid.util.args.DNArgument; 073import com.unboundid.util.args.IntegerArgument; 074import com.unboundid.util.args.StringArgument; 075 076 077 078/** 079 * This class provides a tool that may be used to identify references to entries 080 * that do not exist. This tool can be useful for verifying existing data in 081 * directory servers that provide support for referential integrity. 082 * <BR><BR> 083 * All of the necessary information is provided using command line arguments. 084 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 085 * class, as well as the following additional arguments: 086 * <UL> 087 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use 088 * for the searches. At least one base DN must be provided.</LI> 089 * <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute 090 * that is expected to contain references to other entries. This 091 * attribute should be indexed for equality searches, and its values 092 * should be DNs. At least one attribute must be provided.</LI> 093 * <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search 094 * to find entries with references to other entries should use the simple 095 * paged results control to iterate across entries in fixed-size pages 096 * rather than trying to use a single search to identify all entries that 097 * reference other entries.</LI> 098 * </UL> 099 */ 100@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 101public final class IdentifyReferencesToMissingEntries 102 extends LDAPCommandLineTool 103 implements SearchResultListener 104{ 105 /** 106 * The serial version UID for this serializable class. 107 */ 108 private static final long serialVersionUID = 1981894839719501258L; 109 110 111 112 // The number of entries examined so far. 113 private final AtomicLong entriesExamined; 114 115 // The argument used to specify the base DNs to use for searches. 116 private DNArgument baseDNArgument; 117 118 // The argument used to specify the search page size. 119 private IntegerArgument pageSizeArgument; 120 121 // The connection to use for retrieving referenced entries. 122 private LDAPConnectionPool getReferencedEntriesPool; 123 124 // A map with counts of missing references by attribute type. 125 private final Map<String,AtomicLong> missingReferenceCounts; 126 127 // The names of the attributes for which to find missing references. 128 private String[] attributes; 129 130 // The argument used to specify the attributes for which to find missing 131 // references. 132 private StringArgument attributeArgument; 133 134 135 136 /** 137 * Parse the provided command line arguments and perform the appropriate 138 * processing. 139 * 140 * @param args The command line arguments provided to this program. 141 */ 142 public static void main(final String... args) 143 { 144 final ResultCode resultCode = main(args, System.out, System.err); 145 if (resultCode != ResultCode.SUCCESS) 146 { 147 System.exit(resultCode.intValue()); 148 } 149 } 150 151 152 153 /** 154 * Parse the provided command line arguments and perform the appropriate 155 * processing. 156 * 157 * @param args The command line arguments provided to this program. 158 * @param outStream The output stream to which standard out should be 159 * written. It may be {@code null} if output should be 160 * suppressed. 161 * @param errStream The output stream to which standard error should be 162 * written. It may be {@code null} if error messages 163 * should be suppressed. 164 * 165 * @return A result code indicating whether the processing was successful. 166 */ 167 public static ResultCode main(final String[] args, 168 final OutputStream outStream, 169 final OutputStream errStream) 170 { 171 final IdentifyReferencesToMissingEntries tool = 172 new IdentifyReferencesToMissingEntries(outStream, errStream); 173 return tool.runTool(args); 174 } 175 176 177 178 /** 179 * Creates a new instance of this tool. 180 * 181 * @param outStream The output stream to which standard out should be 182 * written. It may be {@code null} if output should be 183 * suppressed. 184 * @param errStream The output stream to which standard error should be 185 * written. It may be {@code null} if error messages 186 * should be suppressed. 187 */ 188 public IdentifyReferencesToMissingEntries(final OutputStream outStream, 189 final OutputStream errStream) 190 { 191 super(outStream, errStream); 192 193 baseDNArgument = null; 194 pageSizeArgument = null; 195 attributeArgument = null; 196 getReferencedEntriesPool = null; 197 198 entriesExamined = new AtomicLong(0L); 199 missingReferenceCounts = new TreeMap<>(); 200 } 201 202 203 204 /** 205 * Retrieves the name of this tool. It should be the name of the command used 206 * to invoke this tool. 207 * 208 * @return The name for this tool. 209 */ 210 @Override() 211 public String getToolName() 212 { 213 return "identify-references-to-missing-entries"; 214 } 215 216 217 218 /** 219 * Retrieves a human-readable description for this tool. 220 * 221 * @return A human-readable description for this tool. 222 */ 223 @Override() 224 public String getToolDescription() 225 { 226 return "This tool may be used to identify entries containing one or more " + 227 "attributes which reference entries that do not exist. This may " + 228 "require the ability to perform unindexed searches and/or the " + 229 "ability to use the simple paged results control."; 230 } 231 232 233 234 /** 235 * Retrieves a version string for this tool, if available. 236 * 237 * @return A version string for this tool, or {@code null} if none is 238 * available. 239 */ 240 @Override() 241 public String getToolVersion() 242 { 243 return Version.NUMERIC_VERSION_STRING; 244 } 245 246 247 248 /** 249 * Indicates whether this tool should provide support for an interactive mode, 250 * in which the tool offers a mode in which the arguments can be provided in 251 * a text-driven menu rather than requiring them to be given on the command 252 * line. If interactive mode is supported, it may be invoked using the 253 * "--interactive" argument. Alternately, if interactive mode is supported 254 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 255 * interactive mode may be invoked by simply launching the tool without any 256 * arguments. 257 * 258 * @return {@code true} if this tool supports interactive mode, or 259 * {@code false} if not. 260 */ 261 @Override() 262 public boolean supportsInteractiveMode() 263 { 264 return true; 265 } 266 267 268 269 /** 270 * Indicates whether this tool defaults to launching in interactive mode if 271 * the tool is invoked without any command-line arguments. This will only be 272 * used if {@link #supportsInteractiveMode()} returns {@code true}. 273 * 274 * @return {@code true} if this tool defaults to using interactive mode if 275 * launched without any command-line arguments, or {@code false} if 276 * not. 277 */ 278 @Override() 279 public boolean defaultsToInteractiveMode() 280 { 281 return true; 282 } 283 284 285 286 /** 287 * Indicates whether this tool should provide arguments for redirecting output 288 * to a file. If this method returns {@code true}, then the tool will offer 289 * an "--outputFile" argument that will specify the path to a file to which 290 * all standard output and standard error content will be written, and it will 291 * also offer a "--teeToStandardOut" argument that can only be used if the 292 * "--outputFile" argument is present and will cause all output to be written 293 * to both the specified output file and to standard output. 294 * 295 * @return {@code true} if this tool should provide arguments for redirecting 296 * output to a file, or {@code false} if not. 297 */ 298 @Override() 299 protected boolean supportsOutputFile() 300 { 301 return true; 302 } 303 304 305 306 /** 307 * Indicates whether this tool should default to interactively prompting for 308 * the bind password if a password is required but no argument was provided 309 * to indicate how to get the password. 310 * 311 * @return {@code true} if this tool should default to interactively 312 * prompting for the bind password, or {@code false} if not. 313 */ 314 @Override() 315 protected boolean defaultToPromptForBindPassword() 316 { 317 return true; 318 } 319 320 321 322 /** 323 * Indicates whether this tool supports the use of a properties file for 324 * specifying default values for arguments that aren't specified on the 325 * command line. 326 * 327 * @return {@code true} if this tool supports the use of a properties file 328 * for specifying default values for arguments that aren't specified 329 * on the command line, or {@code false} if not. 330 */ 331 @Override() 332 public boolean supportsPropertiesFile() 333 { 334 return true; 335 } 336 337 338 339 /** 340 * Indicates whether the LDAP-specific arguments should include alternate 341 * versions of all long identifiers that consist of multiple words so that 342 * they are available in both camelCase and dash-separated versions. 343 * 344 * @return {@code true} if this tool should provide multiple versions of 345 * long identifiers for LDAP-specific arguments, or {@code false} if 346 * not. 347 */ 348 @Override() 349 protected boolean includeAlternateLongIdentifiers() 350 { 351 return true; 352 } 353 354 355 356 /** 357 * Indicates whether this tool should provide a command-line argument that 358 * allows for low-level SSL debugging. If this returns {@code true}, then an 359 * "--enableSSLDebugging}" argument will be added that sets the 360 * "javax.net.debug" system property to "all" before attempting any 361 * communication. 362 * 363 * @return {@code true} if this tool should offer an "--enableSSLDebugging" 364 * argument, or {@code false} if not. 365 */ 366 @Override() 367 protected boolean supportsSSLDebugging() 368 { 369 return true; 370 } 371 372 373 374 /** 375 * Adds the arguments needed by this command-line tool to the provided 376 * argument parser which are not related to connecting or authenticating to 377 * the directory server. 378 * 379 * @param parser The argument parser to which the arguments should be added. 380 * 381 * @throws ArgumentException If a problem occurs while adding the arguments. 382 */ 383 @Override() 384 public void addNonLDAPArguments(final ArgumentParser parser) 385 throws ArgumentException 386 { 387 String description = "The search base DN(s) to use to find entries with " + 388 "references to other entries. At least one base DN must be " + 389 "specified."; 390 baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}", 391 description); 392 baseDNArgument.addLongIdentifier("base-dn", true); 393 parser.addArgument(baseDNArgument); 394 395 description = "The attribute(s) for which to find missing references. " + 396 "At least one attribute must be specified, and each attribute " + 397 "must be indexed for equality searches and have values which are DNs."; 398 attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}", 399 description); 400 parser.addArgument(attributeArgument); 401 402 description = "The maximum number of entries to retrieve at a time when " + 403 "attempting to find entries with references to other entries. This " + 404 "requires that the authenticated user have permission to use the " + 405 "simple paged results control, but it can avoid problems with the " + 406 "server sending entries too quickly for the client to handle. By " + 407 "default, the simple paged results control will not be used."; 408 pageSizeArgument = 409 new IntegerArgument('z', "simplePageSize", false, 1, "{num}", 410 description, 1, Integer.MAX_VALUE); 411 pageSizeArgument.addLongIdentifier("simple-page-size", true); 412 parser.addArgument(pageSizeArgument); 413 } 414 415 416 417 /** 418 * Retrieves the connection options that should be used for connections that 419 * are created with this command line tool. Subclasses may override this 420 * method to use a custom set of connection options. 421 * 422 * @return The connection options that should be used for connections that 423 * are created with this command line tool. 424 */ 425 @Override() 426 public LDAPConnectionOptions getConnectionOptions() 427 { 428 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 429 430 options.setUseSynchronousMode(true); 431 options.setResponseTimeoutMillis(0L); 432 433 return options; 434 } 435 436 437 438 /** 439 * Performs the core set of processing for this tool. 440 * 441 * @return A result code that indicates whether the processing completed 442 * successfully. 443 */ 444 @Override() 445 public ResultCode doToolProcessing() 446 { 447 // Establish a connection to the target directory server to use for 448 // finding references to entries. 449 final LDAPConnectionPool findReferencesPool; 450 try 451 { 452 findReferencesPool = getConnectionPool(1, 1); 453 findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true); 454 } 455 catch (final LDAPException le) 456 { 457 Debug.debugException(le); 458 err("Unable to establish a connection to the directory server: ", 459 StaticUtils.getExceptionMessage(le)); 460 return le.getResultCode(); 461 } 462 463 try 464 { 465 // Establish a second connection to use for retrieving referenced entries. 466 try 467 { 468 getReferencedEntriesPool = getConnectionPool(1,1); 469 getReferencedEntriesPool. 470 setRetryFailedOperationsDueToInvalidConnections(true); 471 } 472 catch (final LDAPException le) 473 { 474 Debug.debugException(le); 475 err("Unable to establish a connection to the directory server: ", 476 StaticUtils.getExceptionMessage(le)); 477 return le.getResultCode(); 478 } 479 480 481 // Get the set of attributes for which to find missing references. 482 final List<String> attrList = attributeArgument.getValues(); 483 attributes = new String[attrList.size()]; 484 attrList.toArray(attributes); 485 486 487 // Construct a search filter that will be used to find all entries with 488 // references to other entries. 489 final Filter filter; 490 if (attributes.length == 1) 491 { 492 filter = Filter.createPresenceFilter(attributes[0]); 493 missingReferenceCounts.put(attributes[0], new AtomicLong(0L)); 494 } 495 else 496 { 497 final Filter[] orComps = new Filter[attributes.length]; 498 for (int i=0; i < attributes.length; i++) 499 { 500 orComps[i] = Filter.createPresenceFilter(attributes[i]); 501 missingReferenceCounts.put(attributes[i], new AtomicLong(0L)); 502 } 503 filter = Filter.createORFilter(orComps); 504 } 505 506 507 // Iterate across all of the search base DNs and perform searches to find 508 // missing references. 509 for (final DN baseDN : baseDNArgument.getValues()) 510 { 511 ASN1OctetString cookie = null; 512 do 513 { 514 final SearchRequest searchRequest = new SearchRequest(this, 515 baseDN.toString(), SearchScope.SUB, filter, attributes); 516 if (pageSizeArgument.isPresent()) 517 { 518 searchRequest.addControl(new SimplePagedResultsControl( 519 pageSizeArgument.getValue(), cookie, false)); 520 } 521 522 SearchResult searchResult; 523 try 524 { 525 searchResult = findReferencesPool.search(searchRequest); 526 } 527 catch (final LDAPSearchException lse) 528 { 529 Debug.debugException(lse); 530 try 531 { 532 searchResult = findReferencesPool.search(searchRequest); 533 } 534 catch (final LDAPSearchException lse2) 535 { 536 Debug.debugException(lse2); 537 searchResult = lse2.getSearchResult(); 538 } 539 } 540 541 if (searchResult.getResultCode() != ResultCode.SUCCESS) 542 { 543 err("An error occurred while attempting to search for missing " + 544 "references to entries below " + baseDN + ": " + 545 searchResult.getDiagnosticMessage()); 546 return searchResult.getResultCode(); 547 } 548 549 final SimplePagedResultsControl pagedResultsResponse; 550 try 551 { 552 pagedResultsResponse = SimplePagedResultsControl.get(searchResult); 553 } 554 catch (final LDAPException le) 555 { 556 Debug.debugException(le); 557 err("An error occurred while attempting to decode a simple " + 558 "paged results response control in the response to a " + 559 "search for entries below " + baseDN + ": " + 560 StaticUtils.getExceptionMessage(le)); 561 return le.getResultCode(); 562 } 563 564 if (pagedResultsResponse != null) 565 { 566 if (pagedResultsResponse.moreResultsToReturn()) 567 { 568 cookie = pagedResultsResponse.getCookie(); 569 } 570 else 571 { 572 cookie = null; 573 } 574 } 575 } 576 while (cookie != null); 577 } 578 579 580 // See if there were any missing references found. 581 boolean missingReferenceFound = false; 582 for (final Map.Entry<String,AtomicLong> e : 583 missingReferenceCounts.entrySet()) 584 { 585 final long numMissing = e.getValue().get(); 586 if (numMissing > 0L) 587 { 588 if (! missingReferenceFound) 589 { 590 err(); 591 missingReferenceFound = true; 592 } 593 594 err("Found " + numMissing + ' ' + e.getKey() + 595 " references to entries that do not exist."); 596 } 597 } 598 599 if (missingReferenceFound) 600 { 601 return ResultCode.CONSTRAINT_VIOLATION; 602 } 603 else 604 { 605 out("No references were found to entries that do not exist."); 606 return ResultCode.SUCCESS; 607 } 608 } 609 finally 610 { 611 findReferencesPool.close(); 612 613 if (getReferencedEntriesPool != null) 614 { 615 getReferencedEntriesPool.close(); 616 } 617 } 618 } 619 620 621 622 /** 623 * Retrieves a map that correlates the number of missing references found by 624 * attribute type. 625 * 626 * @return A map that correlates the number of missing references found by 627 * attribute type. 628 */ 629 public Map<String,AtomicLong> getMissingReferenceCounts() 630 { 631 return Collections.unmodifiableMap(missingReferenceCounts); 632 } 633 634 635 636 /** 637 * Retrieves a set of information that may be used to generate example usage 638 * information. Each element in the returned map should consist of a map 639 * between an example set of arguments and a string that describes the 640 * behavior of the tool when invoked with that set of arguments. 641 * 642 * @return A set of information that may be used to generate example usage 643 * information. It may be {@code null} or empty if no example usage 644 * information is available. 645 */ 646 @Override() 647 public LinkedHashMap<String[],String> getExampleUsages() 648 { 649 final LinkedHashMap<String[],String> exampleMap = 650 new LinkedHashMap<>(StaticUtils.computeMapCapacity(1)); 651 652 final String[] args = 653 { 654 "--hostname", "server.example.com", 655 "--port", "389", 656 "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com", 657 "--bindPassword", "password", 658 "--baseDN", "dc=example,dc=com", 659 "--attribute", "member", 660 "--attribute", "uniqueMember", 661 "--simplePageSize", "100" 662 }; 663 exampleMap.put(args, 664 "Identify all entries below dc=example,dc=com in which either the " + 665 "member or uniqueMember attribute references an entry that " + 666 "does not exist."); 667 668 return exampleMap; 669 } 670 671 672 673 /** 674 * Indicates that the provided search result entry has been returned by the 675 * server and may be processed by this search result listener. 676 * 677 * @param searchEntry The search result entry that has been returned by the 678 * server. 679 */ 680 @Override() 681 public void searchEntryReturned(final SearchResultEntry searchEntry) 682 { 683 try 684 { 685 // Find attributes which references to entries that do not exist. 686 for (final String attr : attributes) 687 { 688 final List<Attribute> attrList = 689 searchEntry.getAttributesWithOptions(attr, null); 690 for (final Attribute a : attrList) 691 { 692 for (final String value : a.getValues()) 693 { 694 try 695 { 696 final SearchResultEntry e = 697 getReferencedEntriesPool.getEntry(value, "1.1"); 698 if (e == null) 699 { 700 err("Entry '", searchEntry.getDN(), "' includes attribute ", 701 a.getName(), " that references entry '", value, 702 "' which does not exist."); 703 missingReferenceCounts.get(attr).incrementAndGet(); 704 } 705 } 706 catch (final LDAPException le) 707 { 708 Debug.debugException(le); 709 err("An error occurred while attempting to determine whether " + 710 "entry '" + value + "' referenced in attribute " + 711 a.getName() + " of entry '" + searchEntry.getDN() + 712 "' exists: " + StaticUtils.getExceptionMessage(le)); 713 missingReferenceCounts.get(attr).incrementAndGet(); 714 } 715 } 716 } 717 } 718 } 719 finally 720 { 721 final long count = entriesExamined.incrementAndGet(); 722 if ((count % 1000L) == 0L) 723 { 724 out(count, " entries examined"); 725 } 726 } 727 } 728 729 730 731 /** 732 * Indicates that the provided search result reference has been returned by 733 * the server and may be processed by this search result listener. 734 * 735 * @param searchReference The search result reference that has been returned 736 * by the server. 737 */ 738 @Override() 739 public void searchReferenceReturned( 740 final SearchResultReference searchReference) 741 { 742 // No implementation is required. This tool will not follow referrals. 743 } 744}