Category Archives: Programming

Interview preparation: Cheat sheet, and my first multi-threaded application (with timing of how long each version took)

Cheat sheet!

Front:

Back:

My first multi-threaded application: Original single-threaded version

This duplicates a directory structure, where the tab indentation in every text file is replaced with spaces. First, the single-threaded version (this is the same function as in my XBN-Java library, extracted so it has no dependencies on any other part of the library…aside from the timing class).

This took 2,764,557,795 nanoseconds to execute. 2.7 seconds.

package multithreading_tabs_to_spaces;
   import  com.github.xbn.testdev.TimedTest;
   import java.io.BufferedWriter;
   import java.io.Console;
   import java.io.File;
   import java.io.FileWriter;
   import java.io.IOException;
   import java.io.PrintWriter;
   import java.text.DecimalFormat;
   import java.util.Iterator;
   import java.util.Objects;
   import org.apache.commons.io.FileUtils;
   import org.slf4j.Logger;
   import org.slf4j.LoggerFactory;
/**
 * <code>java multithreading_tabs_to_spaces.IndentTabsToSpaces_NonMultiThreaded 0 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation\</code>
 *
 * <code>java -classpath .;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\commons-io-2.4.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-api-1.7.12.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-simple-1.7.12.jar multithreading_tabs_to_spaces.IndentTabsToSpaces_NonMultiThreaded 5 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation</code>
 */
public class IndentTabsToSpaces_NonMultiThreaded  {
   private static final String FILE_SEP = System.getProperty("file.separator", "\\");
   private static final String PARAM_REQUIREMENTS = "Two required parameters: " +
      "Number of y/n overwrite prompts, and the path of the directory containing " +
      "the files to overwrite (must *not* end with a " + FILE_SEP + ".";
   public static final void main(String[] cmd_lineParams) {
      int areYouSureOverwritePromptCount;
      String sourceDirPreDashSrcDst;
      try {
         areYouSureOverwritePromptCount = Integer.parseInt(cmd_lineParams[0]);
         //itr = FileUtils.iterateFiles(new File(cmd_lineParams[1]), FileFilterUtils.trueFileFilter(), null);
         sourceDirPreDashSrcDst = cmd_lineParams[1];
      } catch(ArrayIndexOutOfBoundsException x) {
         throw new RuntimeException(PARAM_REQUIREMENTS);
      } catch(NumberFormatException x)  {
         throw new RuntimeException("Parameter one must be an integer. " +
            PARAM_REQUIREMENTS);
      }

      String sourceDir = sourceDirPreDashSrcDst + "-source" + FILE_SEP;
      Iterator<File> itr = FileUtils.iterateFiles(new File(sourceDir),
         new String[] {"xml", "java", "txt", "bat", "md", "log", "javax"}, true);

      TimedTest singleThreadedTest = new TimedTest("single-threaded");

      singleThreadedTest.declareStartWithOutput();
      try {
         new TabToSpaceIndenter().replaceAllInDirectory(
            areYouSureOverwritePromptCount, itr,
            sourceDir,
            sourceDirPreDashSrcDst + "-destination" + FILE_SEP);
      } catch(IOException x) {
         throw new RuntimeException(x);
      }
      singleThreadedTest.declareEndWithOutput();
   }
}
class TabToSpaceIndenter {
   //State:
      private final String rplcmntSpaces;
      private final Logger logger = LoggerFactory.getLogger("TabToSpaceIndenter");
   //Constants:
      private static final DecimalFormat DEC_FMT = new DecimalFormat("#.###");
      private static final String LINE_SEP = System.getProperty("line.separator", "\r\n");
      private static final boolean OVERWRITE = false;
      private static final boolean MANUAL_FLUSH = false;
      /**
       * The default value to replace each tab with--equal to three spaces.
       * @see #getReplacementSpaces()
       */
      public static final String SPACES_TO_RPLC_WITH_DEFAULT = "   ";
   /**
    * <p>Create a new instance with the default spaces-replacement.</p>
    *
    * <p>Equal to
    * <br> &nbsp; &nbsp; <code>{@link #TabToSpaceIndenter(String) this}(SPACES_TO_RPLC_WITH_DEFAULT)</code></p>
    */
   public TabToSpaceIndenter() {
      this(SPACES_TO_RPLC_WITH_DEFAULT);
   }
   /**
    * Create a new instance.
    * @param   spaces_toRplcWith  The spaces to replace each tab with. May
    * not be {@code null} or empty. Get with
    * {@link #getReplacementSpaces() getReplacementSpaces}{@code ()}.
    * @see #TabToSpaceIndenter()
    */
   public TabToSpaceIndenter(String spaces_toRplcWith) {
      try {
         if(spaces_toRplcWith.length() == 0) {
            throw new IllegalArgumentException("spaces_toRplcWith has no characters.");
         }
      } catch(NullPointerException x) {
         Objects.requireNonNull(spaces_toRplcWith, "spaces_toRplcWith");
         throw x;
      }
      rplcmntSpaces = spaces_toRplcWith;
   }
   /**
    * The value to replace each tab with.
    * @see #TabToSpaceIndenter(String)
    * @see #SPACES_TO_RPLC_WITH_DEFAULT
    */
   public String getReplacementSpaces() {
      return rplcmntSpaces;
   }
   /**
    * <p>Utility function to replace all indentation tabs for all files in
    * a directory--<i><b>this overwrites all files!</b></i></p>
    *
    * @param areYouSure_overwritePrompts If greater than zero, this is the
    * number of times the user is presented with a prompt to confirm
    * overwriting. If five, for instance, the <i>first</i> five files are
    * not overwritten until the user confirms each (by answering with a
    * <code>'Y'</code> or <code>'y'</code>. Anything else aborts the
    * application.
    * @param file_itr May not be {@code null}. <i>Every</i> returned file
    * in this iterator is expected to be a plain-text file (so filter it as
    * necessary), and both readable and writable.
    * <i>Should</i> be non-{@code null} and non-empty.
    * @param source_baseDir The directory base-path in which files are read
    * from. This is what is replaced by <code>dest_baseDir</code>. May not
    * be {@code null} and must begin the path for every file returned by
    * <code>file_itr</code>.
    * @param dest_baseDir The directory base-path that output is written
    * to. This is what <code>source_baseDir</code> is replaced by. May not
    * be {@code null} or empty.
    * @see #getReplacementSpaces()
    * @see  org.apache.commons.io.FileUtils#iterateFiles(File, IOFileFilter, IOFileFilter) commons.io.FileUtils#iterateFiles
    */
   public void replaceAllInDirectory(int areYouSure_overwritePrompts,
            Iterator<File> file_itr, String source_baseDir, String dest_baseDir)
            throws IOException  {
      logger.info("Source dir:      " + source_baseDir);
      logger.info("Destination dir: " + dest_baseDir);

      Objects.requireNonNull(file_itr, "file_itr");

      int fileCount = 0;        //How many total files were analyzed (and
                                //potentially changed)?
      int totalTabsRplcd = 0;   //How many tabs were replaced in *all*
      int aysPromptsGiven = 0;  //files?

      while(file_itr.hasNext()) {
         File f = file_itr.next();
         fileCount++;

         String sourcePath = f.getAbsolutePath();
         if(!sourcePath.startsWith(source_baseDir))  {
            throw new IllegalArgumentException("sourcePath (" + sourcePath +
               ") does not start with source_baseDir (" + source_baseDir + ").");
         }
         String destPath = dest_baseDir + sourcePath.substring(source_baseDir.length());

         if(++aysPromptsGiven < areYouSure_overwritePrompts)  {
            String promptText = "[" + aysPromptsGiven + "/" +
               areYouSure_overwritePrompts + "] About to overwrite" + LINE_SEP +
               "   " + destPath + LINE_SEP + "Are you sure? 'Y' or 'y' to " +
               "proceed. Anything else to abort: ";

            Console console = System.console();
            Objects.requireNonNull(console, "System.console() (This error is " +
               "expected when you run this in Eclipse. This works when " +
               "executing it directly on the console. If you set the first " +
               "parameter to zero, you can safely run it in Eclipse.)");
            String input = console.readLine(promptText);
            if(!input.toLowerCase().equals("y"))  {
               System.out.println("Aborting.");
               return;
            }
         }

         //Replace the tabs in this file, and get the number of tabs
         //actually replaced.
         totalTabsRplcd += replaceAllInFile(f, destPath);
      }

      //Print summary to console.
      String avgTabsPerFileStr = DEC_FMT.format(totalTabsRplcd /
                                                new Double(fileCount));
      logger.info("Done. {} total tabs replaced in {} total files ({}/file)",
         totalTabsRplcd, fileCount, avgTabsPerFileStr, logger);
   }
   public int replaceAllInFile(File source_file, String dest_path) {
      try {
         logger.trace("Getting input line iterator to {}." + source_file.getAbsolutePath());
      } catch(NullPointerException x) {
         Objects.requireNonNull(source_file, "source_file");
         Objects.requireNonNull(logger, "logger");
         throw x;
      }

      Iterator<String> lineItrInput = null;
      try  {
         lineItrInput = FileUtils.lineIterator(source_file);
      } catch(Exception x) {
         throw new RuntimeException(
            "Attempting to obtain line iterator for \"" + source_file.getAbsolutePath() +
            "\"", x);
      }

      logger.debug("Creating output print writer to dest_path (\"{}\").", dest_path);
      PrintWriter writerOut = null;
      try  {
         writerOut = (new PrintWriter(new BufferedWriter(
            new FileWriter(dest_path, OVERWRITE)), MANUAL_FLUSH));
      } catch(Exception x) {
         throw new RuntimeException("Attempting to create a print writer to \"" +
            dest_path + "\"", x);
      }

      int totalLines = 0;     //How many lines in this file?
      int tabsRplcd = 0;      //How many tabs total in *all* lines?

      try  {
         while(lineItrInput.hasNext()) {
            String line = lineItrInput.next();
            totalLines++;

            if(line.length() == 0) {
               //No text at all, so definitely no tabs.
               writerOut.write(LINE_SEP);
               logger.trace("{} No characters. Writing out LINE_SEP only.", totalLines);
               continue;
            }

            //At least some text.

            int charIdx = 0;

            //Starting at the left-most character, while it's a tab,
            //replace it with spaces.
            while(charIdx < line.length() && line.charAt(charIdx) == '\t') {
               //There *is* another character in the line, and it *is* a
               //tab.
               charIdx++;
               writerOut.write(getReplacementSpaces());
               tabsRplcd++;
               logger.trace("{}. {} tabs replaced.", totalLines, tabsRplcd);
            }

            //No more tabs. Append the rest of the line.

            writerOut.write(line.substring(charIdx));
            writerOut.write(LINE_SEP);
            logger.trace("{}. No more tabs in this file. Appending the rest of the line.", totalLines);
         }
      } catch(Exception x) {
         throw new RuntimeException("source_file=" + source_file.getAbsolutePath() + ", dest_path=" + dest_path + "");

      } finally {
         writerOut.flush();
         writerOut.close();
         logger.trace("{}. Writer flushed and closed.", totalLines);
      }


      String avgTabsPerLineStr = DEC_FMT.format(tabsRplcd / new Double(totalLines));
      logger.debug("This file: {} lines, {} tabs, {} average per line", totalLines, tabsRplcd, avgTabsPerLineStr);

      return tabsRplcd;
   }
}

My first multi-threaded application: The same class, changed to multi-threading

This uses one of a number of executors (and how long it took):

  • Single threaded (2,247,164,905 nanoseconds, 2.2 seconds)
  • Fixed thread pool with 2 (958,872,242), 5 (731,709,280), 10 (699,200,429), and 20 threads (704,502,249),
  • Cached thread pool (765,272,723)

So the ten-to-twenty-thread thread pool is the clear winner (the above, original single-threaded version took 2.7 seconds).

package multithreading_tabs_to_spaces;
   import java.io.BufferedWriter;
   import java.io.Console;
   import java.io.File;
   import java.io.FileWriter;
   import java.io.IOException;
   import java.io.PrintWriter;
   import java.text.DecimalFormat;
   import java.util.ArrayList;
   import java.util.concurrent.Callable;
   import java.util.concurrent.ExecutionException;
   import java.util.concurrent.Executors;
   import java.util.concurrent.ExecutorService;
   import java.util.concurrent.Future;
   import java.util.Iterator;
   import java.util.List;
   import java.util.Objects;
   import org.apache.commons.io.FileUtils;
   import org.slf4j.Logger;
   import org.slf4j.LoggerFactory;
   import com.github.xbn.testdev.TimedTest;
/**
 * java multithreading_tabs_to_spaces.IndentTabsToSpaces_NonMultiThreaded 0 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation\
 *
 * java -classpath C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\bin\;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\commons-io-2.4.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-api-1.7.12.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-simple-1.7.12.jar multithreading_tabs_to_spaces.IndentTabsToSpaces_MultiThreaded 5 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation
 */
public class IndentTabsToSpaces_MultiThreaded  {
   private static final String FILE_SEP = System.getProperty("file.separator", "\\");
   private static final String PARAM_REQUIREMENTS = "Two required parameters: " +
      "Number of y/n overwrite prompts, and the path of the directory containing " +
      "the files to overwrite (must *not* end with a " + FILE_SEP + ".";
   public static final void main(String[] cmd_lineParams) {
      int areYouSureOverwritePromptCount;
      String sourceDirPreDashSrcDst;
      try {
         areYouSureOverwritePromptCount = Integer.parseInt(cmd_lineParams[0]);
         //itr = FileUtils.iterateFiles(new File(cmd_lineParams[1]), FileFilterUtils.trueFileFilter(), null);
         sourceDirPreDashSrcDst = cmd_lineParams[1];
      } catch(ArrayIndexOutOfBoundsException x) {
         throw new RuntimeException(PARAM_REQUIREMENTS);
      } catch(NumberFormatException x)  {
         throw new RuntimeException("Parameter one must be an integer. " +
            PARAM_REQUIREMENTS);
      }

      String sourceDir = sourceDirPreDashSrcDst + "-source" + FILE_SEP;
      String destDir = sourceDirPreDashSrcDst + "-destination" + FILE_SEP;
      Iterator<File> itr = FileUtils.iterateFiles(new File(sourceDir),
         new String[] {"xml", "java", "txt", "bat", "md", "log", "javax"}, true);

      TimedTest singleThreadedExecutorTest = new TimedTest("single-threaded executor");

      singleThreadedExecutorTest.declareStartWithOutput();
      replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
         Executors.newSingleThreadExecutor());
      singleThreadedExecutorTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest twoThreadFixedPoolTest = new TimedTest("2 thread fixed pool");
      // twoThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(2));
      // twoThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest fiveThreadFixedPoolTest = new TimedTest("5 thread fixed pool");
      // fiveThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(5));
      // fiveThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest tenThreadFixedPoolTest = new TimedTest("10 thread fixed pool");
      // tenThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(10));
      // tenThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest tenThreadFixedPoolTest = new TimedTest("10 thread fixed pool");
      // tenThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(20));
      // tenThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest cachedPoolTest = new TimedTest("cached pool");
      // cachedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newCachedThreadPool());
      // cachedPoolTest.declareEndWithOutput();
   }
   private static final void pauseOneSecond()  {
      try {
          Thread.sleep(1000);
      } catch(InterruptedException ex) {
          Thread.currentThread().interrupt();
      }
   }
   private static final void replaceAllInDir(Iterator<File> itr, String source_dir, String dest_dir, int ays_promptCount, ExecutorService exec_svc)  {
      try {
         new TabToSpaceIndenterMT().replaceAllInDirectory(ays_promptCount, itr,
            source_dir, dest_dir,
            exec_svc);
      } catch(IOException | InterruptedException x) {
         throw new RuntimeException(x);
      }
   }
}
class TabToSpaceIndenterMT {
   //State:
      private final String rplcmntSpaces;
      private final Logger logger = LoggerFactory.getLogger("TabToSpaceIndenterMT");
   //Constants:
      private static final String LINE_SEP = System.getProperty("line.separator", "\r\n");
      /**
       * The default value to replace each tab with--equal to three spaces.
       * @see #getReplacementSpaces()
       */
      public static final String SPACES_TO_RPLC_WITH_DEFAULT = "   ";
   /**
    * <p>Create a new instance with the default spaces-replacement.</p>
    *
    * <p>Equal to
    * <br> &nbsp; &nbsp; <code>{@link #TabToSpaceIndenterMT(String) this}(SPACES_TO_RPLC_WITH_DEFAULT)</code></p>
    */
   public TabToSpaceIndenterMT() {
      this(SPACES_TO_RPLC_WITH_DEFAULT);
   }
   /**
    * Create a new instance.
    * @param   spaces_toRplcWith  The spaces to replace each tab with. May
    * not be {@code null} or empty. Get with
    * {@link #getReplacementSpaces() getReplacementSpaces}{@code ()}.
    * @see #TabToSpaceIndenterMT()
    */
   public TabToSpaceIndenterMT(String spaces_toRplcWith) {
      try {
         if(spaces_toRplcWith.length() == 0) {
            throw new IllegalArgumentException("spaces_toRplcWith has no characters.");
         }
      } catch(NullPointerException x) {
         Objects.requireNonNull(spaces_toRplcWith, "spaces_toRplcWith");
         throw x;
      }
      rplcmntSpaces = spaces_toRplcWith;
   }
   /**
    * The value to replace each tab with.
    * @see #TabToSpaceIndenterMT(String)
    * @see #SPACES_TO_RPLC_WITH_DEFAULT
    */
   public String getReplacementSpaces() {
      return rplcmntSpaces;
   }
   /**
    * <p>Utility function to replace all indentation tabs for all files in
    * a directory--<i><b>this overwrites all files!</b></i></p>
    *
    * <p>This function prints both <code>INFO</code> and <code>DEBUG</code>
    * level information, via SLF4J.</p>
    *
    * @param areYouSure_overwritePrompts If greater than zero, this is the
    * number of times the user is presented with a prompt to confirm
    * overwriting. If five, for instance, the <i>first</i> five files are
    * not overwritten until the user confirms each (by answering with a
    * <code>'Y'</code> or <code>'y'</code>. Anything else aborts the
    * application.
    * @param file_itr May not be {@code null}. <i>Every</i> returned file
    * in this iterator is expected to be a plain-text file (so filter it as
    * necessary), and both readable and writable.
    * <i>Should</i> be non-{@code null} and non-empty.
    * @param source_baseDir The directory base-path in which files are read
    * from. This is what is replaced by <code>dest_baseDir</code>. May not
    * be {@code null} and must begin the path for every file returned by
    * <code>file_itr</code>.
    * @param dest_baseDir The directory base-path that output is written
    * to. This is what <code>source_baseDir</code> is replaced by. May not
    * be {@code null} or empty.
    * <br>is {@code true}.
    * @see #getReplacementSpaces()
    * @see  org.apache.commons.io.FileUtils#iterateFiles(File, IOFileFilter, IOFileFilter) commons.io.FileUtils#iterateFiles
    */
   public void replaceAllInDirectory(int areYouSure_overwritePrompts,
            Iterator<File> file_itr, String source_baseDir, String dest_baseDir,
            ExecutorService exec_svc) throws IOException, InterruptedException  {
      logger.info("Source dir:      " + source_baseDir);
      logger.info("Destination dir: " + dest_baseDir);

      Objects.requireNonNull(file_itr, "file_itr");

      int fileCount = 0;        //How many total files were analyzed (and
                                //potentially changed)?
      int totalTabsRplcd = 0;   //How many tabs were replaced in *all*
      int aysPromptsGiven = 0;  //files?
      List<Future<Integer>> futureTabsRplcdList = new ArrayList<>(2000);

      while(file_itr.hasNext()) {
         if(Thread.currentThread().isInterrupted())  {
            break;
         }

         File f = file_itr.next();
         fileCount++;

         String sourcePath = f.getAbsolutePath();
         if(!sourcePath.startsWith(source_baseDir))  {
            throw new IllegalArgumentException("sourcePath (" + sourcePath +
               ") does not start with source_baseDir (" + source_baseDir + ").");
         }
         String destPath = dest_baseDir + sourcePath.substring(source_baseDir.length());

         if(++aysPromptsGiven <= areYouSure_overwritePrompts)  {
            String promptText = "[" + aysPromptsGiven + "/" +
               areYouSure_overwritePrompts + "] About to overwrite" + LINE_SEP +
               "   " + destPath + LINE_SEP + "Are you sure? 'Y' or 'y' to " +
               "proceed. Anything else to abort: ";

            Console console = System.console();
            Objects.requireNonNull(console, "System.console() (This error is " +
               "expected when you run this in Eclipse. This works when " +
               "executing it directly on the console. If you set the first " +
               "parameter to zero, you can safely run it in Eclipse.)");
            String input = console.readLine(promptText);
            if(!input.toLowerCase().equals("y"))  {
               System.out.println("Aborting.");
               return;
            }
         }

         //Replace the tabs in this file, and get the number of tabs
         //actually replaced.
         Callable<Integer> rplcAllInFile = new ReplaceAllTabsInOneFile(f, destPath, getReplacementSpaces(), logger);
         Future<Integer> fint = null;
         try {
            fint = exec_svc.submit(rplcAllInFile);
         } catch(NullPointerException npx) {
            throw new NullPointerException("exec_svc");
         }

         //Can't get the result here! Well, you *can*, but then you block until
         //it's ready, on each iteration.
         futureTabsRplcdList.add(fint);
      }
      try {
         for(Future<Integer> fint: futureTabsRplcdList) {
            totalTabsRplcd += fint.get();
         }
      } catch(InterruptedException | ExecutionException x) {
         throw new RuntimeException("Attempting to get the result from the ReplaceAllTabsInOneFile future: " + x);
      }

      //Print summary to console.
      String avgTabsPerFileStr = new DecimalFormat("#.###").format(totalTabsRplcd /
                                                new Double(fileCount));
      logger.info("Done. {} total tabs replaced in {} total files ({}/file)",
         totalTabsRplcd, fileCount, avgTabsPerFileStr, logger);

      if(Thread.currentThread().isInterrupted())  {
         throw new InterruptedException("Interrupted. INFO-level logged summary " +
            "is only for the " + fileCount + " files processed so far.");
      }
   }
}
class ReplaceAllTabsInOneFile implements Callable<Integer> {
   private static final String LINE_SEP = System.getProperty("line.separator", "\r\n");
   private static final boolean OVERWRITE = false;
   private static final boolean MANUAL_FLUSH = false;
   private static final DecimalFormat DEC_FMT = new DecimalFormat("#.###");
   private final File srcFile;
   private final String destPath;
   private final String spaces;
   private final Logger logger;
   public ReplaceAllTabsInOneFile(File source_file, String dest_path, String spcs_toRplcWith, Logger logger)  {
      srcFile = source_file;
      destPath = dest_path;
      spaces = spcs_toRplcWith;
      this.logger = logger;
   }
   public Integer call() {
      try {
         logger.trace("Getting input line iterator to {}." + srcFile.getAbsolutePath());
      } catch(NullPointerException x) {
         Objects.requireNonNull(srcFile, "source_file");
         Objects.requireNonNull(logger, "logger");
         throw x;
      }

      logger.trace("Getting input line iterator to {}." + srcFile.getAbsolutePath());
      Iterator<String> lineItrInput = null;
      try  {
         lineItrInput = FileUtils.lineIterator(srcFile);
      } catch(Exception x) {
         throw new RuntimeException(
            "Attempting to obtain line iterator for \"" + srcFile.getAbsolutePath() +
            "\"", x);
      }

      logger.debug("Creating output print writer to dest_path (\"{}\").", destPath);
      PrintWriter writerOut = null;
      try  {
         writerOut = (new PrintWriter(new BufferedWriter(
            new FileWriter(destPath, OVERWRITE)), MANUAL_FLUSH));
      } catch(Exception x) {
         throw new RuntimeException("Attempting to create a print writer to \"" +
            destPath + "\"", x);
      }

      int totalLines = 0;     //How many lines in this file?
      int tabsRplcd = 0;      //How many tabs total in *all* lines?

      try  {
         while(lineItrInput.hasNext()) {
            String line = lineItrInput.next();
            totalLines++;

            if(line.length() == 0) {
               //No text at all, so definitely no tabs.
               writerOut.write(LINE_SEP);
               logger.trace("{} No characters. Writing out LINE_SEP only.", totalLines);
               continue;
            }

            //At least some text.

            int charIdx = 0;

            //Starting at the left-most character, while it's a tab,
            //replace it with spaces.
            while(charIdx < line.length() && line.charAt(charIdx) == '\t') {
               //There *is* another character in the line, and it *is* a
               //tab.
               charIdx++;
               writerOut.write(spaces);
               tabsRplcd++;
               logger.trace("{}. {} tabs replaced.", totalLines, tabsRplcd);
            }

            //No more tabs. Append the rest of the line.

            writerOut.write(line.substring(charIdx));
            writerOut.write(LINE_SEP);
            logger.trace("{}. No more tabs in this file. Appending the rest of the line.", totalLines);
         }
      } catch(Exception x) {
         throw new RuntimeException("srcFile=" + srcFile.getAbsolutePath() + ", destPath=" + destPath + "");

      } finally {
         writerOut.flush();
         writerOut.close();
         logger.trace("{}. Writer flushed and closed.", totalLines);
      }

      String avgTabsPerLineStr = DEC_FMT.format(tabsRplcd / new Double(totalLines));
      logger.debug("This file: {} lines, {} tabs, {} average per line", totalLines, tabsRplcd, avgTabsPerLineStr);

      return tabsRplcd;
   }
}
Advertisements

Summary of the major non-concurrent, non-synchronized Java collections

Studying up for a job interview. Big emphasis on concurrency in this company, which I’m inexperienced with. Reading Java Concurrency In Practice, and I’m finding myself reviewing the basic collection types over and over, so I’m putting them down here once and for all.

Collection: An interface representing an unordered “bag” of items, called “elements”. The “next” element is undefined (random).

  • Set: An interface representing a Collection with no duplicates.
    • HashSet: A Set backed by a Hashtable. Fastest and smallest memory usage, when ordering is unimportant.
    • LinkedHashSet: A HashSet with the addition of a linked list to associate elements in insertion order. The “next” element is the next-most-recently inserted element.
    • TreeSet: A Set where elements are ordered by a Comparator (typically natural ordering. Slowest and largest memory usage, but necessary for comparator-based ordering.
    • EnumSet: An extremely fast and efficient Set customized for a single enum type.
  • List: An interface representing an Collection whose elements are ordered and each have a numeric index representing its position, where zero is the first element, and (length - 1) is the last.
    • ArrayList: A List backed by an array, where the array has a length (called “capacity”) that is at least as large as the number of elements (the list’s “size”). When size exceeds capacity (when the (capacity + 1)-th element is added), the array is recreated with a new capacity of (new length * 1.5)–this recreation is fast, since it uses System.arrayCopy(). Deleting and inserting/adding elements requires all neighboring elements (to the right) be shifted into or out of that space. Accessing any element is fast, as it only requires the calculation (element-zero-address + desired-index * element-size) to find it’s location. In most situations, an ArrayList is preferred over a LinkedList.
    • LinkedList: A List backed by a set of objects, each linked to its “previous” and “next” neighbors. A LinkedList is also a Queue and Deque. Accessing elements is done starting at the first or last element, and traversing until the desired index is reached. Insertion and deletion, once the desired index is reached via traversal is a trivial matter of re-mapping only the immediate-neighbor links to point to the new element or bypass the now-deleted element.
  • Map: An interface representing an Collection where each element has an identifying “key”–each element is a key-value pair.
    • HashMap: A Map where keys are unordered, and backed by a Hashtable.
    • LinkedhashMap: Keys are ordered by insertion order.
    • TreeMap: A Map where keys are ordered by a Comparator (typically natural ordering).
  • Queue: An interface that represents a Collection where elements are, typically, added to one end, and removed from the other (FIFO: first-in, first-out).
  • Stack: An interface that represents a Collection where elements are, typically, both added (pushed) and removed (popped) from the same end (LIFO: last-in, first-out).
  • Deque: Pronounced “deck”. A linked list that is typically only added to and read from either end (not the middle).

Basic collection diagrams:

Comparing the insertion of an element with an ArrayList and LinkedList:

This is also an answer on Stack Overflow. Please consider up-voting it.

XYplorer custom toolbar button to open a “catalog” along with catalog-specific tabsets

Each item in this custom toolbar button saves the current tabsets, then opens a catalog along with the tabsets from the last time that catalog was opened. If the tabsets don’t exist, they’re created.

Right-clicking does nothing.

Someday I’ll move these to script files, and add in custom toolbar button-sets for each catalog as well.

XYplorer is cool.

Snip: CTB 1
  XYplorer 14.80.0224, 2/20/2015 6:07:17 PM
Action
  NewUserButton
Name
  Open catalog
Icon
  label:Ctlg>000000,ffffff
ScriptL
  "Open catalog: XBN-Java"
     tabset("save", "1"); tabset("save", "2"); tabset("load", "xbn_left_pane", "1"); tabset("load", "xbn_right_pane", "2"); catalogload "git_repositories_xbnjava.dat";
  "Open catalog: Non-XBN-Java repositories"
     tabset("save", "1"); tabset("save", "2"); tabset("load", "nonxbn_left_pane", "1"); tabset("load", "nonxbn_right_pane", "2");    catalogload "git_repositories_non_xbnjava.dat";
  "Open catalog: Django Auth Tutorial"
     tabset("save", "1"); tabset("save", "2"); tabset("load", "Q_left_pane", "1"); tabset("load", "Q_right_pane", "2"); catalogload "django_auth_tutorial.dat";

ScriptR

FireClick
  0

(This is duplicated from the XY-forums.)

The full user-authentication lifecycle in Django, with testing at every step — The step-by-step tutorial I wish I had (part seven)

…This chapter is complete except no tests yet…


In parts one, two, and three, we set up the model and trivial website, and in four, five, and six, a working login page and logout links. In this post, we’re going to add a final feature to the login page: a password reset link, which sends a one-time-use-only link to the user’s email account. When clicked on, they are presented with a set-your-new-password form.

[TOC: one, two, three, four, five, six, seven, eight, nine, ten]

Reset your password: Overview

This is a time-consuming, although straight-forward feature to implement. By far the hardest thing is understanding the sequence of events, particularly because the views are confusingly named. This is how it works (with specifics based on the implementation in this chapter):

  1. On the login page there is an “I forgot my password” link.
  2. Click on it and you are taken to a page on which you need to enter your the email address from when you created the account. Pressing submit sends a one-time-only reset-my-password link to the user’s email. The view for this page is django.contrib.auth.views.password_reset.
  3. After it’s submitted, another page appears whose only purpose is to inform that the the email was sent, and they should go and view it for further instructions. The view for this page is django.contrib.auth.views.password_reset_done.
  4. The email itself is sent. The template for this email is specified by the email_template_name parameter of the password_reset view. See the next section for an example email.
  5. The link in the email takes you to the “set your new password” form, which includes a redundant confirmation field. The view for this page is django.contrib.auth.views.password_reset_confirm.
  6. After the form is submitted, the final page is presented, which only states “your password has been changed”, and likely provides a link back to the login page. The view for this page is django.contrib.auth.views.password_reset_complete.

Although I would choose these views to have clearer names more along the lines of

  • password_reset_1of4_email_request,
  • password_reset_2of4_email_sent,
  • pwd_reset_3of4_new_pwd_form, and
  • password_reset_4of4_finished

the existing names are here to stay. We are going to use the “better” ones as much as possible, though.

Set up email: Print to console only

Normally an email is actually sent (how to do this). Instead, we’re going to print its contents to the console. This is trivially-implemented by adding a single variable to
    /home/myname/django_auth_lifecycle/djauth_root/django_auth_lifecycle/settings.py

# https://docs.djangoproject.com/en/1.7/topics/email/#django.core.mail.backends.smtp.EmailBackend
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

This is what the email looks like when printed to the console, with some empty lines removed (the 127.0.0.1.:8001 in the link must be changed to the name of your webserver):

MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: 7bit
Subject: Password reset on 127.0.0.1:8001
From: webmaster@localhost
To: myemailaddress@yahoo.com
Date: Wed, 18 Feb 2015 16:38:02 -0000
Message-ID: 

You're receiving this email because you requested a password reset for your user account at 127.0.0.1:8001.
Please go to the following page and choose a new password:

http://127.0.0.1:8001/auth/pwd_reset_3of4_new_pwd_form/MQ/3zd-add9dfa05216b9ead4cc/

Your username, in case you've forgotten: admin
Thanks for using our site!
The 127.0.0.1:8001 team
-------------------------------------------------------------------------------

Set the login view name

One more change in the settings file.

In order for the final “your password was successfully changed” view (password_reset_complete) to link back to the login page, we must tell it where to link to, since ours is not using the default name. Add the LOGIN_URL variable:

# https://docs.djangoproject.com/en/1.7/ref/settings/#login-url
LOGIN_URL="login"     #View name in auth_lifecycle.registration.urls

Activate the forgot-my-password link

In
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/templates/registration/login.html

Change

...I forgot my password...

to

<a href="{% url 'password_reset' %}">I forgot my password</a>

Custom templates: Overview

Aside from the email itself, each of the four views has its own template. Creating custom versions of these templates is optional. You could use all of the built-in defaults and skip straight to updating urls.py. The directory containing all default templates, as installed with Django, is:
    /home/myname/django_auth_lifecycle/djauth_venv/lib/python3.4/site-packages/django/contrib/admin/templates/registration/

To create a custom template, you can

  • Duplicate the entire custom template and tweak what you like, or
  • Extend the template and override only the needed sections

Regardless which way you choose, these default templates should be used as a reference if you encounter any problems.

The relative path of the file (as based off of one of the TEMPLATE_DIRS) must be either

  • Equal to the default value of the view’s template_name parameter, as specified in each view’s documentation. For example, the password_reset view states “Defaults to registration/password_change_done.html if not supplied.”
  • Or set to an alternate value, by passing it through it’s url-entry:
    url(r"^pwd_reset_3of4_new_pwd_form/(?P<uidb64>\w+)/(?P<token>[\w-]+)/$",
        "django.contrib.auth.views.password_reset_confirm",
        { "template_name": "registration/pwd_reset_3of4_new_pwd_form.html" }

(The email itself is specified by the email_template_name parameter in password_reset.)

Custom template: pwd_reset_3of4_new_pwd_form

The only custom template we’ll be creating is for the set-your-new-password form (django.contrib.auth.views.password_reset_done), so we can also do a client-side check for the password lengths, and that they’re equal. This will be implemented with JQuery Validation.

This is the default template, as installed by Django:
    /home/myname/django_auth_lifecycle/djauth_venv/lib/python3.4/site-packages/django/contrib/admin/templates/registration/password_reset_confirm.html

{% extends "admin/base_site.html" %}
{% load i18n %}

{% block breadcrumbs %}
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
&rsaquo; {% trans 'Password reset confirmation' %}
</div>
{% endblock %}

{% block title %}{{ title }}{% endblock %}
{% block content_title %}<h1>{{ title }}</h1>{% endblock %}
{% block content %}

{% if validlink %}

<p>{% trans "Please enter your new password twice so we can verify you typed it in correctly." %}</p>

<form action="" method="post">{% csrf_token %}
{{ form.new_password1.errors }}
<p class="aligned wide"><label for="id_new_password1">{% trans 'New password:' %}</label>{{ form.new_password1 }}</p>
{{ form.new_password2.errors }}
<p class="aligned wide"><label for="id_new_password2">{% trans 'Confirm password:' %}</label>{{ form.new_password2 }}</p>
<p><input type="submit" value="{% trans 'Change my password' %}" /></p>
</form>

{% else %}

<p>{% trans "The password reset link was invalid, possibly because it has already been used.  Please request a new password reset." %}</p>

{% endif %}

{% endblock %}

The entire “content” block must be replaced with our custom code. The form as associated to the JavaScript by giving it the name “newPwdForm“.

Save the following as
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/templates/registration/pwd_reset_3of4_new_pwd_form.html

{% extends "registration/password_reset_confirm.html" %}
{% comment %}
   The extends Must be the first line in the template. This comment may
   not be before it.

   Example use:
   - http://stackoverflow.com/a/28570678/2736496
   Documentation:
   - https://docs.djangoproject.com/en/1.7/ref/templates/builtins/#std:templatetag-extends
{% endcomment %}
{% load i18n %}       {# For the "trans" tag #}
{% block content %}

{% if validlink %}

<p>{% trans "Please enter your new password twice so we can verify." %}</p>
<p>{% trans "you typed it in correctly." %}</p>

<form action="" method="post" id="newPwdForm"><!-- Form id rqd by JS. -->
   {% csrf_token %}
   {{ form.new_password1.errors }}
   <p class="aligned wide">
      <label for="id_new_password1">{% trans 'New password:' %}</label>
      {{ form.new_password1 }}</p>
   {{ form.new_password2.errors }}
   <p class="aligned wide">
      <label for="id_new_password2">{% trans 'Confirm password:' %}</label>
      {{ form.new_password2 }}</p>
   <p><input type="submit" value="{% trans 'Change my password' %}" /></p>
</form>

{% else %}

<p>{% trans "The password reset link was invalid, possibly because it" %}
{% trans "has already been used.  Please request a new password reset." %}</p>

{% endif %}

<script src="http://code.jquery.com/jquery-1.11.1.min.js"></script>
<script src="http://jqueryvalidation.org/files/dist/jquery.validate.min.js"></script>
<script src="http://jqueryvalidation.org/files/dist/additional-methods.min.js"></script>
<script>
   //These values come from auth_lifecycle.models
   var minPassLen = {{ PASSWORD_MIN_LEN }}; //PASSWORD_MIN_LEN
   var maxPassLen = {{ PASSWORD_MAX_LEN }}; //PASSWORD_MAX_LEN

   var passwordMsg = "{% trans "Password must be between " %}" + minPassLen +
      "{% trans " and " %}" + maxPassLen +
      "{% trans " characters, inclusive." %}";

   jQuery.validator.setDefaults({
      success: "valid",
      //Avoids form submit. Comment when in production...START
      //debug: true
      //submitHandler: function() {
      //   alert("Success! The form was pretend-submitted!");
      //}
      //Avoids form submit. Comment when in production...END
   });
   $( "#newPwdForm" ).validate({
      rules: {
         new_password1: {
            required: true,
            minlength: minPassLen,
            maxlength: maxPassLen
         },
         new_password2: {
            //http://jqueryvalidation.org/equalTo-method
            equalTo: "#id_new_password1"
         }
      },
      messages:  {
         new_password1: {
            required: "{% trans "Password required" %}",
            minlength: passwordMsg,
            maxlength: passwordMsg
         }
      }
   });
</script>

{% endblock %}

Server-side check

As done with the login form, we’re going to update the set-a-new-password form to enforce length. The default form used by this view, django.contrib.auth.forms.SetPasswordForm, does check for the passwords being equal, but does not have any min or max lengths:

new_password1 = forms.CharField(label=_("New password"),
                                widget=forms.PasswordInput)

Save the following as
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/form_reset_set_new_pwd.py

from auth_lifecycle.models     import PASSWORD_MIN_LEN, PASSWORD_MAX_LEN
from auth_lifecycle.registration.view_login import get_min_max_incl_err_msg
from django                    import forms    #NOT django.contrib.auth.forms
from django.contrib.auth.forms import SetPasswordForm
from django.utils.translation  import ugettext, ugettext_lazy as _

min_max_len_err_msg = get_min_max_incl_err_msg(PASSWORD_MIN_LEN, PASSWORD_MAX_LEN)

class SetPasswordFormEnforceLength(SetPasswordForm):
    """
    A `SetPasswordForm` that enforces min/max lengths.
    - https://docs.djangoproject.com/en/1.7/_modules/django/contrib/auth/forms/#SetPasswordForm

    Pass this into the login form via the `set_password_form` parameter.
    - https://docs.djangoproject.com/en/1.7/topics/auth/default/#django.contrib.auth.views.password_reset_confirm
    Which is done in `registration/urls.py`.
    """
    new_password1 = forms.CharField(label=_("New password"),
                                    widget=forms.PasswordInput,
                                    min_length=PASSWORD_MIN_LEN,
                                    max_length=PASSWORD_MAX_LEN,
                                    error_messages={
                                        'min_length': min_max_len_err_msg,
                                        'max_length': min_max_len_err_msg })

(To repeat the warning from the top of part six: The server-side length checks are “succeeding” but crashing–that is, they only crash when the lengths are incorrect. While this is a critical problem, it only applies when the client-side JavaScript is disabled. Here is a Stack Overflow question documenting the problem. A solution would be greatly appreciated.)

Configure the urls

Replace the contents of
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/urls.py
with

from auth_lifecycle.models import PASSWORD_MIN_LEN, PASSWORD_MAX_LEN
from auth_lifecycle.registration.view_login import AuthenticationFormEnforceLength
from auth_lifecycle.registration.form_reset_set_new_pwd import SetPasswordFormEnforceLength
from django.conf.urls      import patterns, url
#Passing keyword arguments through url entries:
# - https://docs.djangoproject.com/en/1.7/topics/http/urls/#passing-extra-options-to-view-functions
urlpatterns = patterns('',
    url(r"^login/$",
        "auth_lifecycle.registration.view_login.login_maybe_remember",
        { "authentication_form": AuthenticationFormEnforceLength },
        name="login"),
    url(r"^logout_then_login/$", "django.contrib.auth.views.logout_then_login",
        {"login_url": "login"}, name="logout_then_login"),
    url(r"^password_reset_1of4_email_request/$",
        "django.contrib.auth.views.password_reset", name="password_reset"),
    url(r"^password_reset_2of4_email_sent/$",
        "django.contrib.auth.views.password_reset_done",
        name="password_reset_done"),
    url(r"^pwd_reset_3of4_new_pwd_form/(?P<uidb64>\w+)/(?P<token>[\w-]+)/$",
        "django.contrib.auth.views.password_reset_confirm",
        { "template_name": "registration/pwd_reset_3of4_new_pwd_form.html",
          "extra_context": { "PASSWORD_MIN_LEN": PASSWORD_MIN_LEN,
                             "PASSWORD_MAX_LEN": PASSWORD_MAX_LEN },
          "set_password_form": SetPasswordFormEnforceLength },
        name="password_reset_confirm"),
    #If NOT using a custom template:
    # url(r"^pwd_reset_3of4_new_pwd_form/(?P<uidb64>\w+)/(?P<token>[\w-]+)/$",
    #     "django.contrib.auth.views.password_reset_confirm",
    #     name="password_reset_confirm"),
    url(r"^password_reset_4of4_finished/$",
        "django.contrib.auth.views.password_reset_complete",
        name="password_reset_complete" ),
)

Tests

Save the following as

    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/....py

...

Output:


Our tests passed.

Give it a try!

Follow these steps to start your server. A reminder to check your console for the text of email.

In the next post, we move on to the change your password form. After that, the final steps are creating and deleting an account.

[TOC: one, two, three, four, five, six, seven, eight, nine, ten]

…to be continued…

(cue cliffhanger segue music)