Category Archives: Programming

Interview preparation: Cheat sheet, and my first multi-threaded application (with timing of how long each version took)

Cheat sheet!

Front:

Back:

My first multi-threaded application: Original single-threaded version

This duplicates a directory structure, where the tab indentation in every text file is replaced with spaces. First, the single-threaded version (this is the same function as in my XBN-Java library, extracted so it has no dependencies on any other part of the library…aside from the timing class).

This took 2,764,557,795 nanoseconds to execute. 2.7 seconds.

package multithreading_tabs_to_spaces;
   import  com.github.xbn.testdev.TimedTest;
   import java.io.BufferedWriter;
   import java.io.Console;
   import java.io.File;
   import java.io.FileWriter;
   import java.io.IOException;
   import java.io.PrintWriter;
   import java.text.DecimalFormat;
   import java.util.Iterator;
   import java.util.Objects;
   import org.apache.commons.io.FileUtils;
   import org.slf4j.Logger;
   import org.slf4j.LoggerFactory;
/**
 * <code>java multithreading_tabs_to_spaces.IndentTabsToSpaces_NonMultiThreaded 0 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation\</code>
 *
 * <code>java -classpath .;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\commons-io-2.4.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-api-1.7.12.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-simple-1.7.12.jar multithreading_tabs_to_spaces.IndentTabsToSpaces_NonMultiThreaded 5 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation</code>
 */
public class IndentTabsToSpaces_NonMultiThreaded  {
   private static final String FILE_SEP = System.getProperty("file.separator", "\\");
   private static final String PARAM_REQUIREMENTS = "Two required parameters: " +
      "Number of y/n overwrite prompts, and the path of the directory containing " +
      "the files to overwrite (must *not* end with a " + FILE_SEP + ".";
   public static final void main(String[] cmd_lineParams) {
      int areYouSureOverwritePromptCount;
      String sourceDirPreDashSrcDst;
      try {
         areYouSureOverwritePromptCount = Integer.parseInt(cmd_lineParams[0]);
         //itr = FileUtils.iterateFiles(new File(cmd_lineParams[1]), FileFilterUtils.trueFileFilter(), null);
         sourceDirPreDashSrcDst = cmd_lineParams[1];
      } catch(ArrayIndexOutOfBoundsException x) {
         throw new RuntimeException(PARAM_REQUIREMENTS);
      } catch(NumberFormatException x)  {
         throw new RuntimeException("Parameter one must be an integer. " +
            PARAM_REQUIREMENTS);
      }

      String sourceDir = sourceDirPreDashSrcDst + "-source" + FILE_SEP;
      Iterator<File> itr = FileUtils.iterateFiles(new File(sourceDir),
         new String[] {"xml", "java", "txt", "bat", "md", "log", "javax"}, true);

      TimedTest singleThreadedTest = new TimedTest("single-threaded");

      singleThreadedTest.declareStartWithOutput();
      try {
         new TabToSpaceIndenter().replaceAllInDirectory(
            areYouSureOverwritePromptCount, itr,
            sourceDir,
            sourceDirPreDashSrcDst + "-destination" + FILE_SEP);
      } catch(IOException x) {
         throw new RuntimeException(x);
      }
      singleThreadedTest.declareEndWithOutput();
   }
}
class TabToSpaceIndenter {
   //State:
      private final String rplcmntSpaces;
      private final Logger logger = LoggerFactory.getLogger("TabToSpaceIndenter");
   //Constants:
      private static final DecimalFormat DEC_FMT = new DecimalFormat("#.###");
      private static final String LINE_SEP = System.getProperty("line.separator", "\r\n");
      private static final boolean OVERWRITE = false;
      private static final boolean MANUAL_FLUSH = false;
      /**
       * The default value to replace each tab with--equal to three spaces.
       * @see #getReplacementSpaces()
       */
      public static final String SPACES_TO_RPLC_WITH_DEFAULT = "   ";
   /**
    * <p>Create a new instance with the default spaces-replacement.</p>
    *
    * <p>Equal to
    * <br> &nbsp; &nbsp; <code>{@link #TabToSpaceIndenter(String) this}(SPACES_TO_RPLC_WITH_DEFAULT)</code></p>
    */
   public TabToSpaceIndenter() {
      this(SPACES_TO_RPLC_WITH_DEFAULT);
   }
   /**
    * Create a new instance.
    * @param   spaces_toRplcWith  The spaces to replace each tab with. May
    * not be {@code null} or empty. Get with
    * {@link #getReplacementSpaces() getReplacementSpaces}{@code ()}.
    * @see #TabToSpaceIndenter()
    */
   public TabToSpaceIndenter(String spaces_toRplcWith) {
      try {
         if(spaces_toRplcWith.length() == 0) {
            throw new IllegalArgumentException("spaces_toRplcWith has no characters.");
         }
      } catch(NullPointerException x) {
         Objects.requireNonNull(spaces_toRplcWith, "spaces_toRplcWith");
         throw x;
      }
      rplcmntSpaces = spaces_toRplcWith;
   }
   /**
    * The value to replace each tab with.
    * @see #TabToSpaceIndenter(String)
    * @see #SPACES_TO_RPLC_WITH_DEFAULT
    */
   public String getReplacementSpaces() {
      return rplcmntSpaces;
   }
   /**
    * <p>Utility function to replace all indentation tabs for all files in
    * a directory--<i><b>this overwrites all files!</b></i></p>
    *
    * @param areYouSure_overwritePrompts If greater than zero, this is the
    * number of times the user is presented with a prompt to confirm
    * overwriting. If five, for instance, the <i>first</i> five files are
    * not overwritten until the user confirms each (by answering with a
    * <code>'Y'</code> or <code>'y'</code>. Anything else aborts the
    * application.
    * @param file_itr May not be {@code null}. <i>Every</i> returned file
    * in this iterator is expected to be a plain-text file (so filter it as
    * necessary), and both readable and writable.
    * <i>Should</i> be non-{@code null} and non-empty.
    * @param source_baseDir The directory base-path in which files are read
    * from. This is what is replaced by <code>dest_baseDir</code>. May not
    * be {@code null} and must begin the path for every file returned by
    * <code>file_itr</code>.
    * @param dest_baseDir The directory base-path that output is written
    * to. This is what <code>source_baseDir</code> is replaced by. May not
    * be {@code null} or empty.
    * @see #getReplacementSpaces()
    * @see  org.apache.commons.io.FileUtils#iterateFiles(File, IOFileFilter, IOFileFilter) commons.io.FileUtils#iterateFiles
    */
   public void replaceAllInDirectory(int areYouSure_overwritePrompts,
            Iterator<File> file_itr, String source_baseDir, String dest_baseDir)
            throws IOException  {
      logger.info("Source dir:      " + source_baseDir);
      logger.info("Destination dir: " + dest_baseDir);

      Objects.requireNonNull(file_itr, "file_itr");

      int fileCount = 0;        //How many total files were analyzed (and
                                //potentially changed)?
      int totalTabsRplcd = 0;   //How many tabs were replaced in *all*
      int aysPromptsGiven = 0;  //files?

      while(file_itr.hasNext()) {
         File f = file_itr.next();
         fileCount++;

         String sourcePath = f.getAbsolutePath();
         if(!sourcePath.startsWith(source_baseDir))  {
            throw new IllegalArgumentException("sourcePath (" + sourcePath +
               ") does not start with source_baseDir (" + source_baseDir + ").");
         }
         String destPath = dest_baseDir + sourcePath.substring(source_baseDir.length());

         if(++aysPromptsGiven < areYouSure_overwritePrompts)  {
            String promptText = "[" + aysPromptsGiven + "/" +
               areYouSure_overwritePrompts + "] About to overwrite" + LINE_SEP +
               "   " + destPath + LINE_SEP + "Are you sure? 'Y' or 'y' to " +
               "proceed. Anything else to abort: ";

            Console console = System.console();
            Objects.requireNonNull(console, "System.console() (This error is " +
               "expected when you run this in Eclipse. This works when " +
               "executing it directly on the console. If you set the first " +
               "parameter to zero, you can safely run it in Eclipse.)");
            String input = console.readLine(promptText);
            if(!input.toLowerCase().equals("y"))  {
               System.out.println("Aborting.");
               return;
            }
         }

         //Replace the tabs in this file, and get the number of tabs
         //actually replaced.
         totalTabsRplcd += replaceAllInFile(f, destPath);
      }

      //Print summary to console.
      String avgTabsPerFileStr = DEC_FMT.format(totalTabsRplcd /
                                                new Double(fileCount));
      logger.info("Done. {} total tabs replaced in {} total files ({}/file)",
         totalTabsRplcd, fileCount, avgTabsPerFileStr, logger);
   }
   public int replaceAllInFile(File source_file, String dest_path) {
      try {
         logger.trace("Getting input line iterator to {}." + source_file.getAbsolutePath());
      } catch(NullPointerException x) {
         Objects.requireNonNull(source_file, "source_file");
         Objects.requireNonNull(logger, "logger");
         throw x;
      }

      Iterator<String> lineItrInput = null;
      try  {
         lineItrInput = FileUtils.lineIterator(source_file);
      } catch(Exception x) {
         throw new RuntimeException(
            "Attempting to obtain line iterator for \"" + source_file.getAbsolutePath() +
            "\"", x);
      }

      logger.debug("Creating output print writer to dest_path (\"{}\").", dest_path);
      PrintWriter writerOut = null;
      try  {
         writerOut = (new PrintWriter(new BufferedWriter(
            new FileWriter(dest_path, OVERWRITE)), MANUAL_FLUSH));
      } catch(Exception x) {
         throw new RuntimeException("Attempting to create a print writer to \"" +
            dest_path + "\"", x);
      }

      int totalLines = 0;     //How many lines in this file?
      int tabsRplcd = 0;      //How many tabs total in *all* lines?

      try  {
         while(lineItrInput.hasNext()) {
            String line = lineItrInput.next();
            totalLines++;

            if(line.length() == 0) {
               //No text at all, so definitely no tabs.
               writerOut.write(LINE_SEP);
               logger.trace("{} No characters. Writing out LINE_SEP only.", totalLines);
               continue;
            }

            //At least some text.

            int charIdx = 0;

            //Starting at the left-most character, while it's a tab,
            //replace it with spaces.
            while(charIdx < line.length() && line.charAt(charIdx) == '\t') {
               //There *is* another character in the line, and it *is* a
               //tab.
               charIdx++;
               writerOut.write(getReplacementSpaces());
               tabsRplcd++;
               logger.trace("{}. {} tabs replaced.", totalLines, tabsRplcd);
            }

            //No more tabs. Append the rest of the line.

            writerOut.write(line.substring(charIdx));
            writerOut.write(LINE_SEP);
            logger.trace("{}. No more tabs in this file. Appending the rest of the line.", totalLines);
         }
      } catch(Exception x) {
         throw new RuntimeException("source_file=" + source_file.getAbsolutePath() + ", dest_path=" + dest_path + "");

      } finally {
         writerOut.flush();
         writerOut.close();
         logger.trace("{}. Writer flushed and closed.", totalLines);
      }


      String avgTabsPerLineStr = DEC_FMT.format(tabsRplcd / new Double(totalLines));
      logger.debug("This file: {} lines, {} tabs, {} average per line", totalLines, tabsRplcd, avgTabsPerLineStr);

      return tabsRplcd;
   }
}

My first multi-threaded application: The same class, changed to multi-threading

This uses one of a number of executors (and how long it took):

  • Single threaded (2,247,164,905 nanoseconds, 2.2 seconds)
  • Fixed thread pool with 2 (958,872,242), 5 (731,709,280), 10 (699,200,429), and 20 threads (704,502,249),
  • Cached thread pool (765,272,723)

So the ten-to-twenty-thread thread pool is the clear winner (the above, original single-threaded version took 2.7 seconds).

package multithreading_tabs_to_spaces;
   import java.io.BufferedWriter;
   import java.io.Console;
   import java.io.File;
   import java.io.FileWriter;
   import java.io.IOException;
   import java.io.PrintWriter;
   import java.text.DecimalFormat;
   import java.util.ArrayList;
   import java.util.concurrent.Callable;
   import java.util.concurrent.ExecutionException;
   import java.util.concurrent.Executors;
   import java.util.concurrent.ExecutorService;
   import java.util.concurrent.Future;
   import java.util.Iterator;
   import java.util.List;
   import java.util.Objects;
   import org.apache.commons.io.FileUtils;
   import org.slf4j.Logger;
   import org.slf4j.LoggerFactory;
   import com.github.xbn.testdev.TimedTest;
/**
 * java multithreading_tabs_to_spaces.IndentTabsToSpaces_NonMultiThreaded 0 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation\
 *
 * java -classpath C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\bin\;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\commons-io-2.4.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-api-1.7.12.jar;C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\jar_dependencies\slf4j-simple-1.7.12.jar multithreading_tabs_to_spaces.IndentTabsToSpaces_MultiThreaded 5 C:\data_jeffy\code\wordpress_posts\java\multithreading_tabs_to_spaces\xbnjava_my_private_sandbox_with_tab_indentation
 */
public class IndentTabsToSpaces_MultiThreaded  {
   private static final String FILE_SEP = System.getProperty("file.separator", "\\");
   private static final String PARAM_REQUIREMENTS = "Two required parameters: " +
      "Number of y/n overwrite prompts, and the path of the directory containing " +
      "the files to overwrite (must *not* end with a " + FILE_SEP + ".";
   public static final void main(String[] cmd_lineParams) {
      int areYouSureOverwritePromptCount;
      String sourceDirPreDashSrcDst;
      try {
         areYouSureOverwritePromptCount = Integer.parseInt(cmd_lineParams[0]);
         //itr = FileUtils.iterateFiles(new File(cmd_lineParams[1]), FileFilterUtils.trueFileFilter(), null);
         sourceDirPreDashSrcDst = cmd_lineParams[1];
      } catch(ArrayIndexOutOfBoundsException x) {
         throw new RuntimeException(PARAM_REQUIREMENTS);
      } catch(NumberFormatException x)  {
         throw new RuntimeException("Parameter one must be an integer. " +
            PARAM_REQUIREMENTS);
      }

      String sourceDir = sourceDirPreDashSrcDst + "-source" + FILE_SEP;
      String destDir = sourceDirPreDashSrcDst + "-destination" + FILE_SEP;
      Iterator<File> itr = FileUtils.iterateFiles(new File(sourceDir),
         new String[] {"xml", "java", "txt", "bat", "md", "log", "javax"}, true);

      TimedTest singleThreadedExecutorTest = new TimedTest("single-threaded executor");

      singleThreadedExecutorTest.declareStartWithOutput();
      replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
         Executors.newSingleThreadExecutor());
      singleThreadedExecutorTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest twoThreadFixedPoolTest = new TimedTest("2 thread fixed pool");
      // twoThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(2));
      // twoThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest fiveThreadFixedPoolTest = new TimedTest("5 thread fixed pool");
      // fiveThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(5));
      // fiveThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest tenThreadFixedPoolTest = new TimedTest("10 thread fixed pool");
      // tenThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(10));
      // tenThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest tenThreadFixedPoolTest = new TimedTest("10 thread fixed pool");
      // tenThreadFixedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newFixedThreadPool(20));
      // tenThreadFixedPoolTest.declareEndWithOutput();

      // pauseOneSecond();

      // TimedTest cachedPoolTest = new TimedTest("cached pool");
      // cachedPoolTest.declareStartWithOutput();
      // replaceAllInDir(itr, sourceDir, destDir, areYouSureOverwritePromptCount,
      //    Executors.newCachedThreadPool());
      // cachedPoolTest.declareEndWithOutput();
   }
   private static final void pauseOneSecond()  {
      try {
          Thread.sleep(1000);
      } catch(InterruptedException ex) {
          Thread.currentThread().interrupt();
      }
   }
   private static final void replaceAllInDir(Iterator<File> itr, String source_dir, String dest_dir, int ays_promptCount, ExecutorService exec_svc)  {
      try {
         new TabToSpaceIndenterMT().replaceAllInDirectory(ays_promptCount, itr,
            source_dir, dest_dir,
            exec_svc);
      } catch(IOException | InterruptedException x) {
         throw new RuntimeException(x);
      }
   }
}
class TabToSpaceIndenterMT {
   //State:
      private final String rplcmntSpaces;
      private final Logger logger = LoggerFactory.getLogger("TabToSpaceIndenterMT");
   //Constants:
      private static final String LINE_SEP = System.getProperty("line.separator", "\r\n");
      /**
       * The default value to replace each tab with--equal to three spaces.
       * @see #getReplacementSpaces()
       */
      public static final String SPACES_TO_RPLC_WITH_DEFAULT = "   ";
   /**
    * <p>Create a new instance with the default spaces-replacement.</p>
    *
    * <p>Equal to
    * <br> &nbsp; &nbsp; <code>{@link #TabToSpaceIndenterMT(String) this}(SPACES_TO_RPLC_WITH_DEFAULT)</code></p>
    */
   public TabToSpaceIndenterMT() {
      this(SPACES_TO_RPLC_WITH_DEFAULT);
   }
   /**
    * Create a new instance.
    * @param   spaces_toRplcWith  The spaces to replace each tab with. May
    * not be {@code null} or empty. Get with
    * {@link #getReplacementSpaces() getReplacementSpaces}{@code ()}.
    * @see #TabToSpaceIndenterMT()
    */
   public TabToSpaceIndenterMT(String spaces_toRplcWith) {
      try {
         if(spaces_toRplcWith.length() == 0) {
            throw new IllegalArgumentException("spaces_toRplcWith has no characters.");
         }
      } catch(NullPointerException x) {
         Objects.requireNonNull(spaces_toRplcWith, "spaces_toRplcWith");
         throw x;
      }
      rplcmntSpaces = spaces_toRplcWith;
   }
   /**
    * The value to replace each tab with.
    * @see #TabToSpaceIndenterMT(String)
    * @see #SPACES_TO_RPLC_WITH_DEFAULT
    */
   public String getReplacementSpaces() {
      return rplcmntSpaces;
   }
   /**
    * <p>Utility function to replace all indentation tabs for all files in
    * a directory--<i><b>this overwrites all files!</b></i></p>
    *
    * <p>This function prints both <code>INFO</code> and <code>DEBUG</code>
    * level information, via SLF4J.</p>
    *
    * @param areYouSure_overwritePrompts If greater than zero, this is the
    * number of times the user is presented with a prompt to confirm
    * overwriting. If five, for instance, the <i>first</i> five files are
    * not overwritten until the user confirms each (by answering with a
    * <code>'Y'</code> or <code>'y'</code>. Anything else aborts the
    * application.
    * @param file_itr May not be {@code null}. <i>Every</i> returned file
    * in this iterator is expected to be a plain-text file (so filter it as
    * necessary), and both readable and writable.
    * <i>Should</i> be non-{@code null} and non-empty.
    * @param source_baseDir The directory base-path in which files are read
    * from. This is what is replaced by <code>dest_baseDir</code>. May not
    * be {@code null} and must begin the path for every file returned by
    * <code>file_itr</code>.
    * @param dest_baseDir The directory base-path that output is written
    * to. This is what <code>source_baseDir</code> is replaced by. May not
    * be {@code null} or empty.
    * <br>is {@code true}.
    * @see #getReplacementSpaces()
    * @see  org.apache.commons.io.FileUtils#iterateFiles(File, IOFileFilter, IOFileFilter) commons.io.FileUtils#iterateFiles
    */
   public void replaceAllInDirectory(int areYouSure_overwritePrompts,
            Iterator<File> file_itr, String source_baseDir, String dest_baseDir,
            ExecutorService exec_svc) throws IOException, InterruptedException  {
      logger.info("Source dir:      " + source_baseDir);
      logger.info("Destination dir: " + dest_baseDir);

      Objects.requireNonNull(file_itr, "file_itr");

      int fileCount = 0;        //How many total files were analyzed (and
                                //potentially changed)?
      int totalTabsRplcd = 0;   //How many tabs were replaced in *all*
      int aysPromptsGiven = 0;  //files?
      List<Future<Integer>> futureTabsRplcdList = new ArrayList<>(2000);

      while(file_itr.hasNext()) {
         if(Thread.currentThread().isInterrupted())  {
            break;
         }

         File f = file_itr.next();
         fileCount++;

         String sourcePath = f.getAbsolutePath();
         if(!sourcePath.startsWith(source_baseDir))  {
            throw new IllegalArgumentException("sourcePath (" + sourcePath +
               ") does not start with source_baseDir (" + source_baseDir + ").");
         }
         String destPath = dest_baseDir + sourcePath.substring(source_baseDir.length());

         if(++aysPromptsGiven <= areYouSure_overwritePrompts)  {
            String promptText = "[" + aysPromptsGiven + "/" +
               areYouSure_overwritePrompts + "] About to overwrite" + LINE_SEP +
               "   " + destPath + LINE_SEP + "Are you sure? 'Y' or 'y' to " +
               "proceed. Anything else to abort: ";

            Console console = System.console();
            Objects.requireNonNull(console, "System.console() (This error is " +
               "expected when you run this in Eclipse. This works when " +
               "executing it directly on the console. If you set the first " +
               "parameter to zero, you can safely run it in Eclipse.)");
            String input = console.readLine(promptText);
            if(!input.toLowerCase().equals("y"))  {
               System.out.println("Aborting.");
               return;
            }
         }

         //Replace the tabs in this file, and get the number of tabs
         //actually replaced.
         Callable<Integer> rplcAllInFile = new ReplaceAllTabsInOneFile(f, destPath, getReplacementSpaces(), logger);
         Future<Integer> fint = null;
         try {
            fint = exec_svc.submit(rplcAllInFile);
         } catch(NullPointerException npx) {
            throw new NullPointerException("exec_svc");
         }

         //Can't get the result here! Well, you *can*, but then you block until
         //it's ready, on each iteration.
         futureTabsRplcdList.add(fint);
      }
      try {
         for(Future<Integer> fint: futureTabsRplcdList) {
            totalTabsRplcd += fint.get();
         }
      } catch(InterruptedException | ExecutionException x) {
         throw new RuntimeException("Attempting to get the result from the ReplaceAllTabsInOneFile future: " + x);
      }

      //Print summary to console.
      String avgTabsPerFileStr = new DecimalFormat("#.###").format(totalTabsRplcd /
                                                new Double(fileCount));
      logger.info("Done. {} total tabs replaced in {} total files ({}/file)",
         totalTabsRplcd, fileCount, avgTabsPerFileStr, logger);

      if(Thread.currentThread().isInterrupted())  {
         throw new InterruptedException("Interrupted. INFO-level logged summary " +
            "is only for the " + fileCount + " files processed so far.");
      }
   }
}
class ReplaceAllTabsInOneFile implements Callable<Integer> {
   private static final String LINE_SEP = System.getProperty("line.separator", "\r\n");
   private static final boolean OVERWRITE = false;
   private static final boolean MANUAL_FLUSH = false;
   private static final DecimalFormat DEC_FMT = new DecimalFormat("#.###");
   private final File srcFile;
   private final String destPath;
   private final String spaces;
   private final Logger logger;
   public ReplaceAllTabsInOneFile(File source_file, String dest_path, String spcs_toRplcWith, Logger logger)  {
      srcFile = source_file;
      destPath = dest_path;
      spaces = spcs_toRplcWith;
      this.logger = logger;
   }
   public Integer call() {
      try {
         logger.trace("Getting input line iterator to {}." + srcFile.getAbsolutePath());
      } catch(NullPointerException x) {
         Objects.requireNonNull(srcFile, "source_file");
         Objects.requireNonNull(logger, "logger");
         throw x;
      }

      logger.trace("Getting input line iterator to {}." + srcFile.getAbsolutePath());
      Iterator<String> lineItrInput = null;
      try  {
         lineItrInput = FileUtils.lineIterator(srcFile);
      } catch(Exception x) {
         throw new RuntimeException(
            "Attempting to obtain line iterator for \"" + srcFile.getAbsolutePath() +
            "\"", x);
      }

      logger.debug("Creating output print writer to dest_path (\"{}\").", destPath);
      PrintWriter writerOut = null;
      try  {
         writerOut = (new PrintWriter(new BufferedWriter(
            new FileWriter(destPath, OVERWRITE)), MANUAL_FLUSH));
      } catch(Exception x) {
         throw new RuntimeException("Attempting to create a print writer to \"" +
            destPath + "\"", x);
      }

      int totalLines = 0;     //How many lines in this file?
      int tabsRplcd = 0;      //How many tabs total in *all* lines?

      try  {
         while(lineItrInput.hasNext()) {
            String line = lineItrInput.next();
            totalLines++;

            if(line.length() == 0) {
               //No text at all, so definitely no tabs.
               writerOut.write(LINE_SEP);
               logger.trace("{} No characters. Writing out LINE_SEP only.", totalLines);
               continue;
            }

            //At least some text.

            int charIdx = 0;

            //Starting at the left-most character, while it's a tab,
            //replace it with spaces.
            while(charIdx < line.length() && line.charAt(charIdx) == '\t') {
               //There *is* another character in the line, and it *is* a
               //tab.
               charIdx++;
               writerOut.write(spaces);
               tabsRplcd++;
               logger.trace("{}. {} tabs replaced.", totalLines, tabsRplcd);
            }

            //No more tabs. Append the rest of the line.

            writerOut.write(line.substring(charIdx));
            writerOut.write(LINE_SEP);
            logger.trace("{}. No more tabs in this file. Appending the rest of the line.", totalLines);
         }
      } catch(Exception x) {
         throw new RuntimeException("srcFile=" + srcFile.getAbsolutePath() + ", destPath=" + destPath + "");

      } finally {
         writerOut.flush();
         writerOut.close();
         logger.trace("{}. Writer flushed and closed.", totalLines);
      }

      String avgTabsPerLineStr = DEC_FMT.format(tabsRplcd / new Double(totalLines));
      logger.debug("This file: {} lines, {} tabs, {} average per line", totalLines, tabsRplcd, avgTabsPerLineStr);

      return tabsRplcd;
   }
}

Summary of the major non-concurrent, non-synchronized Java collections

Studying up for a job interview. Big emphasis on concurrency in this company, which I’m inexperienced with. Reading Java Concurrency In Practice, and I’m finding myself reviewing the basic collection types over and over, so I’m putting them down here once and for all.

Collection: An interface representing an unordered “bag” of items, called “elements”. The “next” element is undefined (random).

  • Set: An interface representing a Collection with no duplicates.
    • HashSet: A Set backed by a Hashtable. Fastest and smallest memory usage, when ordering is unimportant.
    • LinkedHashSet: A HashSet with the addition of a linked list to associate elements in insertion order. The “next” element is the next-most-recently inserted element.
    • TreeSet: A Set where elements are ordered by a Comparator (typically natural ordering. Slowest and largest memory usage, but necessary for comparator-based ordering.
    • EnumSet: An extremely fast and efficient Set customized for a single enum type.
  • List: An interface representing an Collection whose elements are ordered and each have a numeric index representing its position, where zero is the first element, and (length - 1) is the last.
    • ArrayList: A List backed by an array, where the array has a length (called “capacity”) that is at least as large as the number of elements (the list’s “size”). When size exceeds capacity (when the (capacity + 1)-th element is added), the array is recreated with a new capacity of (new length * 1.5)–this recreation is fast, since it uses System.arrayCopy(). Deleting and inserting/adding elements requires all neighboring elements (to the right) be shifted into or out of that space. Accessing any element is fast, as it only requires the calculation (element-zero-address + desired-index * element-size) to find it’s location. In most situations, an ArrayList is preferred over a LinkedList.
    • LinkedList: A List backed by a set of objects, each linked to its “previous” and “next” neighbors. A LinkedList is also a Queue and Deque. Accessing elements is done starting at the first or last element, and traversing until the desired index is reached. Insertion and deletion, once the desired index is reached via traversal is a trivial matter of re-mapping only the immediate-neighbor links to point to the new element or bypass the now-deleted element.
  • Map: An interface representing an Collection where each element has an identifying “key”–each element is a key-value pair.
    • HashMap: A Map where keys are unordered, and backed by a Hashtable.
    • LinkedhashMap: Keys are ordered by insertion order.
    • TreeMap: A Map where keys are ordered by a Comparator (typically natural ordering).
  • Queue: An interface that represents a Collection where elements are, typically, added to one end, and removed from the other (FIFO: first-in, first-out).
  • Stack: An interface that represents a Collection where elements are, typically, both added (pushed) and removed (popped) from the same end (LIFO: last-in, first-out).
  • Deque: Pronounced “deck”. A linked list that is typically only added to and read from either end (not the middle).

Basic collection diagrams:

Comparing the insertion of an element with an ArrayList and LinkedList:

This is also an answer on Stack Overflow. Please consider up-voting it.

XYplorer custom toolbar button to open a “catalog” along with catalog-specific tabsets

Each item in this custom toolbar button saves the current tabsets, then opens a catalog along with the tabsets from the last time that catalog was opened. If the tabsets don’t exist, they’re created.

Right-clicking does nothing.

Someday I’ll move these to script files, and add in custom toolbar button-sets for each catalog as well.

XYplorer is cool.

Snip: CTB 1
  XYplorer 14.80.0224, 2/20/2015 6:07:17 PM
Action
  NewUserButton
Name
  Open catalog
Icon
  label:Ctlg>000000,ffffff
ScriptL
  "Open catalog: XBN-Java"
     tabset("save", "1"); tabset("save", "2"); tabset("load", "xbn_left_pane", "1"); tabset("load", "xbn_right_pane", "2"); catalogload "git_repositories_xbnjava.dat";
  "Open catalog: Non-XBN-Java repositories"
     tabset("save", "1"); tabset("save", "2"); tabset("load", "nonxbn_left_pane", "1"); tabset("load", "nonxbn_right_pane", "2");    catalogload "git_repositories_non_xbnjava.dat";
  "Open catalog: Django Auth Tutorial"
     tabset("save", "1"); tabset("save", "2"); tabset("load", "Q_left_pane", "1"); tabset("load", "Q_right_pane", "2"); catalogload "django_auth_tutorial.dat";

ScriptR

FireClick
  0

(This is duplicated from the XY-forums.)

The full user-authentication lifecycle in Django, with testing at every step — The step-by-step tutorial I wish I had (part seven)

…This chapter is complete except no tests yet…


In parts one, two, and three, we set up the model and trivial website, and in four, five, and six, a working login page and logout links. In this post, we’re going to add a final feature to the login page: a password reset link, which sends a one-time-use-only link to the user’s email account. When clicked on, they are presented with a set-your-new-password form.

[TOC: one, two, three, four, five, six, seven, eight, nine, ten]

Reset your password: Overview

This is a time-consuming, although straight-forward feature to implement. By far the hardest thing is understanding the sequence of events, particularly because the views are confusingly named. This is how it works (with specifics based on the implementation in this chapter):

  1. On the login page there is an “I forgot my password” link.
  2. Click on it and you are taken to a page on which you need to enter your the email address from when you created the account. Pressing submit sends a one-time-only reset-my-password link to the user’s email. The view for this page is django.contrib.auth.views.password_reset.
  3. After it’s submitted, another page appears whose only purpose is to inform that the the email was sent, and they should go and view it for further instructions. The view for this page is django.contrib.auth.views.password_reset_done.
  4. The email itself is sent. The template for this email is specified by the email_template_name parameter of the password_reset view. See the next section for an example email.
  5. The link in the email takes you to the “set your new password” form, which includes a redundant confirmation field. The view for this page is django.contrib.auth.views.password_reset_confirm.
  6. After the form is submitted, the final page is presented, which only states “your password has been changed”, and likely provides a link back to the login page. The view for this page is django.contrib.auth.views.password_reset_complete.

Although I would choose these views to have clearer names more along the lines of

  • password_reset_1of4_email_request,
  • password_reset_2of4_email_sent,
  • pwd_reset_3of4_new_pwd_form, and
  • password_reset_4of4_finished

the existing names are here to stay. We are going to use the “better” ones as much as possible, though.

Set up email: Print to console only

Normally an email is actually sent (how to do this). Instead, we’re going to print its contents to the console. This is trivially-implemented by adding a single variable to
    /home/myname/django_auth_lifecycle/djauth_root/django_auth_lifecycle/settings.py

# https://docs.djangoproject.com/en/1.7/topics/email/#django.core.mail.backends.smtp.EmailBackend
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

This is what the email looks like when printed to the console, with some empty lines removed (the 127.0.0.1.:8001 in the link must be changed to the name of your webserver):

MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: 7bit
Subject: Password reset on 127.0.0.1:8001
From: webmaster@localhost
To: myemailaddress@yahoo.com
Date: Wed, 18 Feb 2015 16:38:02 -0000
Message-ID: 

You're receiving this email because you requested a password reset for your user account at 127.0.0.1:8001.
Please go to the following page and choose a new password:

http://127.0.0.1:8001/auth/pwd_reset_3of4_new_pwd_form/MQ/3zd-add9dfa05216b9ead4cc/

Your username, in case you've forgotten: admin
Thanks for using our site!
The 127.0.0.1:8001 team
-------------------------------------------------------------------------------

Set the login view name

One more change in the settings file.

In order for the final “your password was successfully changed” view (password_reset_complete) to link back to the login page, we must tell it where to link to, since ours is not using the default name. Add the LOGIN_URL variable:

# https://docs.djangoproject.com/en/1.7/ref/settings/#login-url
LOGIN_URL="login"     #View name in auth_lifecycle.registration.urls

Activate the forgot-my-password link

In
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/templates/registration/login.html

Change

...I forgot my password...

to

<a href="{% url 'password_reset' %}">I forgot my password</a>

Custom templates: Overview

Aside from the email itself, each of the four views has its own template. Creating custom versions of these templates is optional. You could use all of the built-in defaults and skip straight to updating urls.py. The directory containing all default templates, as installed with Django, is:
    /home/myname/django_auth_lifecycle/djauth_venv/lib/python3.4/site-packages/django/contrib/admin/templates/registration/

To create a custom template, you can

  • Duplicate the entire custom template and tweak what you like, or
  • Extend the template and override only the needed sections

Regardless which way you choose, these default templates should be used as a reference if you encounter any problems.

The relative path of the file (as based off of one of the TEMPLATE_DIRS) must be either

  • Equal to the default value of the view’s template_name parameter, as specified in each view’s documentation. For example, the password_reset view states “Defaults to registration/password_change_done.html if not supplied.”
  • Or set to an alternate value, by passing it through it’s url-entry:
    url(r"^pwd_reset_3of4_new_pwd_form/(?P<uidb64>\w+)/(?P<token>[\w-]+)/$",
        "django.contrib.auth.views.password_reset_confirm",
        { "template_name": "registration/pwd_reset_3of4_new_pwd_form.html" }

(The email itself is specified by the email_template_name parameter in password_reset.)

Custom template: pwd_reset_3of4_new_pwd_form

The only custom template we’ll be creating is for the set-your-new-password form (django.contrib.auth.views.password_reset_done), so we can also do a client-side check for the password lengths, and that they’re equal. This will be implemented with JQuery Validation.

This is the default template, as installed by Django:
    /home/myname/django_auth_lifecycle/djauth_venv/lib/python3.4/site-packages/django/contrib/admin/templates/registration/password_reset_confirm.html

{% extends "admin/base_site.html" %}
{% load i18n %}

{% block breadcrumbs %}
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
&rsaquo; {% trans 'Password reset confirmation' %}
</div>
{% endblock %}

{% block title %}{{ title }}{% endblock %}
{% block content_title %}<h1>{{ title }}</h1>{% endblock %}
{% block content %}

{% if validlink %}

<p>{% trans "Please enter your new password twice so we can verify you typed it in correctly." %}</p>

<form action="" method="post">{% csrf_token %}
{{ form.new_password1.errors }}
<p class="aligned wide"><label for="id_new_password1">{% trans 'New password:' %}</label>{{ form.new_password1 }}</p>
{{ form.new_password2.errors }}
<p class="aligned wide"><label for="id_new_password2">{% trans 'Confirm password:' %}</label>{{ form.new_password2 }}</p>
<p><input type="submit" value="{% trans 'Change my password' %}" /></p>
</form>

{% else %}

<p>{% trans "The password reset link was invalid, possibly because it has already been used.  Please request a new password reset." %}</p>

{% endif %}

{% endblock %}

The entire “content” block must be replaced with our custom code. The form as associated to the JavaScript by giving it the name “newPwdForm“.

Save the following as
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/templates/registration/pwd_reset_3of4_new_pwd_form.html

{% extends "registration/password_reset_confirm.html" %}
{% comment %}
   The extends Must be the first line in the template. This comment may
   not be before it.

   Example use:
   - http://stackoverflow.com/a/28570678/2736496
   Documentation:
   - https://docs.djangoproject.com/en/1.7/ref/templates/builtins/#std:templatetag-extends
{% endcomment %}
{% load i18n %}       {# For the "trans" tag #}
{% block content %}

{% if validlink %}

<p>{% trans "Please enter your new password twice so we can verify." %}</p>
<p>{% trans "you typed it in correctly." %}</p>

<form action="" method="post" id="newPwdForm"><!-- Form id rqd by JS. -->
   {% csrf_token %}
   {{ form.new_password1.errors }}
   <p class="aligned wide">
      <label for="id_new_password1">{% trans 'New password:' %}</label>
      {{ form.new_password1 }}</p>
   {{ form.new_password2.errors }}
   <p class="aligned wide">
      <label for="id_new_password2">{% trans 'Confirm password:' %}</label>
      {{ form.new_password2 }}</p>
   <p><input type="submit" value="{% trans 'Change my password' %}" /></p>
</form>

{% else %}

<p>{% trans "The password reset link was invalid, possibly because it" %}
{% trans "has already been used.  Please request a new password reset." %}</p>

{% endif %}

<script src="http://code.jquery.com/jquery-1.11.1.min.js"></script>
<script src="http://jqueryvalidation.org/files/dist/jquery.validate.min.js"></script>
<script src="http://jqueryvalidation.org/files/dist/additional-methods.min.js"></script>
<script>
   //These values come from auth_lifecycle.models
   var minPassLen = {{ PASSWORD_MIN_LEN }}; //PASSWORD_MIN_LEN
   var maxPassLen = {{ PASSWORD_MAX_LEN }}; //PASSWORD_MAX_LEN

   var passwordMsg = "{% trans "Password must be between " %}" + minPassLen +
      "{% trans " and " %}" + maxPassLen +
      "{% trans " characters, inclusive." %}";

   jQuery.validator.setDefaults({
      success: "valid",
      //Avoids form submit. Comment when in production...START
      //debug: true
      //submitHandler: function() {
      //   alert("Success! The form was pretend-submitted!");
      //}
      //Avoids form submit. Comment when in production...END
   });
   $( "#newPwdForm" ).validate({
      rules: {
         new_password1: {
            required: true,
            minlength: minPassLen,
            maxlength: maxPassLen
         },
         new_password2: {
            //http://jqueryvalidation.org/equalTo-method
            equalTo: "#id_new_password1"
         }
      },
      messages:  {
         new_password1: {
            required: "{% trans "Password required" %}",
            minlength: passwordMsg,
            maxlength: passwordMsg
         }
      }
   });
</script>

{% endblock %}

Server-side check

As done with the login form, we’re going to update the set-a-new-password form to enforce length. The default form used by this view, django.contrib.auth.forms.SetPasswordForm, does check for the passwords being equal, but does not have any min or max lengths:

new_password1 = forms.CharField(label=_("New password"),
                                widget=forms.PasswordInput)

Save the following as
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/form_reset_set_new_pwd.py

from auth_lifecycle.models     import PASSWORD_MIN_LEN, PASSWORD_MAX_LEN
from auth_lifecycle.registration.view_login import get_min_max_incl_err_msg
from django                    import forms    #NOT django.contrib.auth.forms
from django.contrib.auth.forms import SetPasswordForm
from django.utils.translation  import ugettext, ugettext_lazy as _

min_max_len_err_msg = get_min_max_incl_err_msg(PASSWORD_MIN_LEN, PASSWORD_MAX_LEN)

class SetPasswordFormEnforceLength(SetPasswordForm):
    """
    A `SetPasswordForm` that enforces min/max lengths.
    - https://docs.djangoproject.com/en/1.7/_modules/django/contrib/auth/forms/#SetPasswordForm

    Pass this into the login form via the `set_password_form` parameter.
    - https://docs.djangoproject.com/en/1.7/topics/auth/default/#django.contrib.auth.views.password_reset_confirm
    Which is done in `registration/urls.py`.
    """
    new_password1 = forms.CharField(label=_("New password"),
                                    widget=forms.PasswordInput,
                                    min_length=PASSWORD_MIN_LEN,
                                    max_length=PASSWORD_MAX_LEN,
                                    error_messages={
                                        'min_length': min_max_len_err_msg,
                                        'max_length': min_max_len_err_msg })

(To repeat the warning from the top of part six: The server-side length checks are “succeeding” but crashing–that is, they only crash when the lengths are incorrect. While this is a critical problem, it only applies when the client-side JavaScript is disabled. Here is a Stack Overflow question documenting the problem. A solution would be greatly appreciated.)

Configure the urls

Replace the contents of
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/urls.py
with

from auth_lifecycle.models import PASSWORD_MIN_LEN, PASSWORD_MAX_LEN
from auth_lifecycle.registration.view_login import AuthenticationFormEnforceLength
from auth_lifecycle.registration.form_reset_set_new_pwd import SetPasswordFormEnforceLength
from django.conf.urls      import patterns, url
#Passing keyword arguments through url entries:
# - https://docs.djangoproject.com/en/1.7/topics/http/urls/#passing-extra-options-to-view-functions
urlpatterns = patterns('',
    url(r"^login/$",
        "auth_lifecycle.registration.view_login.login_maybe_remember",
        { "authentication_form": AuthenticationFormEnforceLength },
        name="login"),
    url(r"^logout_then_login/$", "django.contrib.auth.views.logout_then_login",
        {"login_url": "login"}, name="logout_then_login"),
    url(r"^password_reset_1of4_email_request/$",
        "django.contrib.auth.views.password_reset", name="password_reset"),
    url(r"^password_reset_2of4_email_sent/$",
        "django.contrib.auth.views.password_reset_done",
        name="password_reset_done"),
    url(r"^pwd_reset_3of4_new_pwd_form/(?P<uidb64>\w+)/(?P<token>[\w-]+)/$",
        "django.contrib.auth.views.password_reset_confirm",
        { "template_name": "registration/pwd_reset_3of4_new_pwd_form.html",
          "extra_context": { "PASSWORD_MIN_LEN": PASSWORD_MIN_LEN,
                             "PASSWORD_MAX_LEN": PASSWORD_MAX_LEN },
          "set_password_form": SetPasswordFormEnforceLength },
        name="password_reset_confirm"),
    #If NOT using a custom template:
    # url(r"^pwd_reset_3of4_new_pwd_form/(?P<uidb64>\w+)/(?P<token>[\w-]+)/$",
    #     "django.contrib.auth.views.password_reset_confirm",
    #     name="password_reset_confirm"),
    url(r"^password_reset_4of4_finished/$",
        "django.contrib.auth.views.password_reset_complete",
        name="password_reset_complete" ),
)

Tests

Save the following as

    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/....py

...

Output:


Our tests passed.

Give it a try!

Follow these steps to start your server. A reminder to check your console for the text of email.

In the next post, we move on to the change your password form. After that, the final steps are creating and deleting an account.

[TOC: one, two, three, four, five, six, seven, eight, nine, ten]

…to be continued…

(cue cliffhanger segue music)

The full user-authentication lifecycle in Django, with testing at every step — The step-by-step tutorial I wish I had (part six)

In parts one, two, and three, we set up the model and trivial website, and in four and five, a login page with remember-me functionality. In this post, we’re going to upgrade it further, with a basic client-side (JavaScript) check to prevent logins from ever reaching the server when the username or password have an obviously-incorrect length. These min-max values will originate in the model code, and will be passed to the template by the view.

[TOC: one, two, three, four, five, six, seven, eight, nine, ten]

Two warnings:

  • As stated at the top of part one, I am unable to test these JavaScript features with Selenium, due to limitations in my server; it’s command-line only–non-GUI.
  • The server-side length checks are “succeeding” but crashing–that is, they only crash when the lengths are incorrect. While this is a critical problem, it only applies when the client-side JavaScript is disabled. Here is a Stack Overflow question documenting the problem. A solution would be greatly appreciated.

This post is the first to contain a substantial amount of JavaScript. It is therefore time to create a public static web server directory, into which the JavaScript will be placed.

  • The Nginx config file for this project is /etc/nginx/sites-available/django_auth_lifecycle
  • The paths to the Django-root and Virtualenv directories:
    • /home/myname/django_auth_lifecycle/djauth_root/
    • /home/myname/django_auth_lifecycle/djauth_venv/

Update the models

Were going to add four configuration variables that will be used by both the Django code and in the templates, that define the minimum and maximum lengths for both the username and password. Although the model itself will remain the same, models.py is an appropriate place to have these configuration variables.

Replace the contents of
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/models.py
with

"""
Defines a single extra user-profile field for the user-authentication
lifecycle demo project, and defines the absolute minimum and maximum
allowable username and password lengths.

The extra field: Birth year, which must be between <link to MIN_BIRTH_YEAR>
and <link to MAX_BIRTH_YEAR>, inclusive.
"""
from datetime                   import datetime
from django.contrib.auth.models import User
from django.core.exceptions     import ValidationError
from django.db                  import models

USERNAME_MIN_LEN = 5
"""
The database allows one-character usernames. We're going to forbid
anything less than five characters.
"""
USERNAME_MAX_LEN = User._meta.get_field('username').max_length
"""
The maximum allowable username length, as determined by the database
column. Equal to
    `User._meta.get_field('username').max_length`
"""
PASSWORD_MIN_LEN = 5
"""
The password is stored in the database, not as plain text, but as it's
generated hash. Length is therefore not enforced by the database at all.
We're going to minimally protect users against themselves and impose an
five character minimum. (For real, I'd make this eight. When testing, I
make the password equal to the username, so it's temporarily shorter.)
"""
PASSWORD_MAX_LEN = 4096
"""
Imposing a maximum password length is not recommended:
- http://stackoverflow.com/questions/98768/should-i-impose-a-maximum-length-on-passwords

However, Django prevents an attack vector by forbidding excessively-long
passwords (See "Issue: denial-of-service via large passwords"):
- https://www.djangoproject.com/weblog/2013/sep/15/security/
"""

OLDEST_EVER_AGE     = 127  #:Equal to `127`
YOUNGEST_ALLOWED_IN_SYSTEM_AGE = 13   #:Equal to `13`
MAX_BIRTH_YEAR      = datetime.now().year - YOUNGEST_ALLOWED_IN_SYSTEM_AGE
"""Most recent allowed birth year for (youngest) users."""
MIN_BIRTH_YEAR      = datetime.now().year - OLDEST_EVER_AGE
"""Most distant allowed birth year for (oldest) users."""

def _validate_birth_year(birth_year_str):
    """Validator for <link to UserProfile.birth_year>, ensuring the
        selected year is between <link to OLDEST_EVER_AGE> and
        <link to MAX_BIRTH_YEAR>, inclusive.
        Raises:
            ValidationError: When the selected year is invalid.

        - https://docs.djangoproject.com/en/1.7/ref/validators/

        I am a recovered Hungarian Notation junkie (I come from Java). I
        stopped using it long before I started with Python. In this
        particular function, however, because of the necessary cast, it's
        appropriate.
    """
    birth_year_int = -1
    try:
        birth_year_int = int(str(birth_year_str).strip())
    except TypeError:
        raise ValidationError(u'"{0}" is not an integer'.format(birth_year_str))

    if  not (MIN_BIRTH_YEAR <= birth_year_int <= MAX_BIRTH_YEAR):
        message = (u'{0} is an invalid birth year.'
                   u'Must be between {1} and {2}, inclusive')
        raise ValidationError(message.format(
            birth_year_str, MIN_BIRTH_YEAR, MAX_BIRTH_YEAR))
    #It's all good.

class UserProfile(models.Model):
    """Extra information about a user: Birth year.

    ---NOTES---

    Useful related SQL:
    - `select id from auth_user where username <> 'admin';`
    - `select * from auth_lifecycle_userprofile where user_id=(x,x,...);`
    """
    # This line is required. Links UserProfile to a User model instance.
    user = models.OneToOneField(User, related_name="profile")

    # The additional attributes we wish to include.
    birth_year = models.IntegerField(
        blank=True,
        verbose_name="Year you were born",
        validators=[_validate_birth_year])

    # Override the __str__() method to return out something meaningful
    def __str__(self):
        return self.user.username

Update the view

The login view created in the previous chapter now needs to be updated so it pass those configuration variables to the template. These variables will be used by the client-side JavaScript. (An alternative to passing extra context to the template is via urls.py, as demonstrated in the next chapter.)

However, since we have no control over our users’ machines–and therefore cannot know if JavaScript is even enabled–it is also necessary to do this check on the server. The default form used by this view, django.contrib.auth.forms.AuthenticationForm, needs to be updated with our min/max lengths. Here are the original fields:

username = forms.CharField(max_length=254)
password = forms.CharField(label=_("Password"), widget=forms.PasswordInput)

Replace the contents of
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/view_login.py
with:

from auth_lifecycle.models     import PASSWORD_MIN_LEN, PASSWORD_MAX_LEN
from auth_lifecycle.models     import USERNAME_MIN_LEN, USERNAME_MAX_LEN
from django                    import forms    #NOT django.contrib.auth.forms
from django.contrib.auth.forms import AuthenticationForm
from django.contrib.auth.views import login
#from django.core.exceptions    import ValidationError
from django.utils.translation  import ugettext, ugettext_lazy as _

def login_maybe_remember(request, *args, **kwargs):
    """
    Login with remember-me functionality and length checking. If the
    remember-me checkbox is checked, the session is remembered for
    SESSION_COOKIE_AGE seconds. If unchecked, the session expires at
    browser close.

    - https://docs.djangoproject.com/en/1.7/ref/settings/#std:setting-SESSION_COOKIE_AGE
    - https://docs.djangoproject.com/en/1.7/topics/http/sessions/#django.contrib.sessions.backends.base.SessionBase.set_expiry
    - https://docs.djangoproject.com/en/1.7/topics/http/sessions/#django.contrib.sessions.backends.base.SessionBase.get_expire_at_browser_close
    """
    if request.method == 'POST' and not request.POST.get('remember', None):
        #This is a login attempt and the checkbox is not checked.
        request.session.set_expiry(0)

    context = {}
    context["USERNAME_MIN_LEN"] = USERNAME_MIN_LEN
    context["USERNAME_MAX_LEN"] = USERNAME_MAX_LEN
    context["PASSWORD_MIN_LEN"] = PASSWORD_MIN_LEN
    context["PASSWORD_MAX_LEN"] = PASSWORD_MAX_LEN
    kwargs["extra_context"] = context

    print("authentication_form=" + str(kwargs["authentication_form"]));

    return login(request, *args, **kwargs)

def get_min_max_incl_err_msg(min_int, max_int):
    """A basic error message for inclusive string length."""
    "Must be between " + str(min_int) + " and " + str(max_int) + " characters, inclusive."

username_min_max_len_err_msg = get_min_max_incl_err_msg(USERNAME_MIN_LEN, USERNAME_MAX_LEN)
pwd_min_max_len_err_msg = get_min_max_incl_err_msg(PASSWORD_MIN_LEN, PASSWORD_MAX_LEN)

class AuthenticationFormEnforceLength(AuthenticationForm):
    """
    An `AuthenticationForm` that enforces min/max lengths.
    - https://docs.djangoproject.com/en/1.7/_modules/django/contrib/auth/forms/#AuthenticationForm

    Pass this into the login form via the `authentication_form` parameter.
    - https://docs.djangoproject.com/en/1.7/topics/auth/default/#django.contrib.auth.views.login
    Which is done in `registration/urls.py`.
    """
    username = forms.CharField(min_length=USERNAME_MIN_LEN,
                               max_length=USERNAME_MAX_LEN,
                               error_messages={
                                   'min_length': username_min_max_len_err_msg,
                                   'max_length': username_min_max_len_err_msg })
    password = forms.CharField(label=_("Password"), widget=forms.PasswordInput,
                                    min_length=PASSWORD_MIN_LEN,
                                    max_length=PASSWORD_MAX_LEN,
                                    error_messages={
                                        'min_length': pwd_min_max_len_err_msg,
                                        'max_length': pwd_min_max_len_err_msg })
#    def clean(self):
#        raise ValidationError("Yikes")

(A reminder of the second warning at the top of this chapter…)

The bounds in this form object cause maxlength attributes to be placed directly onto the html form elements:

<input id="id_username" maxlength="30" name="username" type="text" /></p>
<input id="id_password" maxlength="4096" name="password" type="password" /></p>

While this now makes the maximum portion of our JavaScript checks redundant, we’re going to leave them in anyway.

Update the template

Now to use these variables in the template.

Replace the contents of
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/templates/registration/login.html
with

{% load i18n %}       {# For the "trans" tag #}
{% load staticfiles %}{# References the static directory.             #}
<!DOCTYPE html>       {# Use with "{% static 'color_ajax_like.js' %}" #}
<html lang="en">
<HTML><HEAD>
    <TITLE>Login</TITLE>
    <!-- The following line makes this page pleasant to view on any device. -->
    <meta name="viewport" content="width=device-width" />
</HEAD>

<BODY>

<H1>Login</H1>

<form method="post" id="loginForm" action="{% url 'login' %}">
{% csrf_token %}
{{ form.as_p }}

   <label><input name="remember" type="checkbox">{% trans "Remember me" %}</label>

   <input type="submit" value="login" />
   <input type="hidden" name="next" value="{% url 'main_page' %}" />
</form>

<P>{% trans "...I forgot my password..., ...Create a new account..." %}</P>

<p><i><a href="{% url 'main_page' %}">View the main page without logging in.</a></i></p>

<script language="JavaScript">
   /*
      Before our JavaScript can be imported, the following variables need
      to be set from some Django variables. While these values could be
      hard-coded here, into the JavaScript, this allows the configuration
      to be centrally located.

      These four values come from auth_lifecycle.models and are required
      by validate_login_user_pass.js.
    */
   var minUserLen = {{ USERNAME_MIN_LEN }}; //USERNAME_MIN_LEN
   var maxUserLen = {{ USERNAME_MAX_LEN }}; //USERNAME_MAX_LEN
   var minPassLen = {{ PASSWORD_MIN_LEN }}; //PASSWORD_MIN_LEN
   var maxPassLen = {{ PASSWORD_MAX_LEN }}; //PASSWORD_MAX_LEN

   document.getElementById("id_username").focus();
</script>
<script src="http://code.jquery.com/jquery-1.11.1.min.js"></script>
<script src="http://ajax.aspnetcdn.com/ajax/jquery.validate/1.13.0/jquery.validate.min.js"></script>
<script type='text/javascript' src="{% static 'js/validate_login_user_pass.js' %}"></script>

</BODY></HTML>

JavaScript validation code

We’ll be using the JQuery Validation Plugin. Here is a separate post about the plugin, where I demonstrate a basic use–the concepts of which are used here.

Save the following as
    /home/myname/django_auth_lifecycle/djauth_root/static/js/validate_login_user_pass.js
(create the "js" sub-directory).

//Configuration...START
   //The following variables are required to exist before this file is
   //loaded: minUserLen, maxUserLen, minPassLen, maxPassLen
   //
   //In addition, the id of the login <form> must be "loginForm"

   var usernameMsg = "Username must be between " + minUserLen + " and " +
                     maxUserLen + " characters, inclusive.";
   var passwordMsg = "Password must be between " + minPassLen + " and " +
                     maxPassLen + " characters, inclusive.";
   jQuery.validator.setDefaults({
      success: "valid",
      //Avoids form submit. Comment when in production...START
      // debug: true,
      // submitHandler: function() {
      //    alert("Success! The form was pretend-submitted!");
      // }
      //Avoids form submit. Comment when in production...END
   });
//Configuration...END
var validateLoginForm = function()  {
   // validate signup form on keyup and submit
   var config = {
      rules: {
         username: {
            required: true,
            minlength: minUserLen,
            maxlength: maxUserLen
         },
         password: {
            required: true,
            minlength: minPassLen,
            maxlength: maxPassLen
         },
      },
      messages: {
         username: {
            required: "Username required",
            minlength: usernameMsg,
            maxlength: usernameMsg
         },
         password: {
            required: "Password required",
            minlength: passwordMsg,
            maxlength: passwordMsg
         }
      }
   };
   $("#loginForm").validate(config);
}
$(document).ready(validateLoginForm);

Tests: Not much to do

Webtest directly accesses the WSGI application server, and therefore cannot test client-side JavaScript. The only thing we’re going to look for is that the variables actually made it into the template, and are being assigned to JavaScript variables as required by the above code.

First, we need a new simple utility testing function. Add this import statement to the top of
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/test__utilities.py

import re

and this function at the bottom:

def assert_pattern_in_string(test_instance, pattern, to_search):
    """
    Asserts that a pattern is found somewhere in a string. This calls

        `test_instance.assertIsNotNone(re.search(pattern, to_search))`

    A failure results in the full pattern and to-search strings being
    printed.
    """
    match = re.search(pattern, to_search)
    test_instance.assertIsNotNone(match,
        "pattern=" + pattern + ", to_search=" + to_search)

Now to create the test-proper. Save the following as
    /home/myname/django_auth_lifecycle/djauth_root/auth_lifecycle/registration/test_login_user_pass_len_vars.py

"""
Tests to confirm that the username and password min/max length variables
are correctly passed through from the view, and assigned as expected.
django-webtest cannot test client-side JavaScript, because it only runs
through the WSGI application server.
- http://stackoverflow.com/a/12552319/2736496
- http://webtest.readthedocs.org/en/latest/#what-this-does

DEPENDS ON TEST:     test__utilities.py
DEPENDED ON BY TEST: None

To run the tests in this file:
    1. source /home/myname/django_files/django_auth_lifecycle/djauth_venv/bin/activate
    2. cd /home/myname/django_files/django_auth_lifecycle/djauth_root/
    3. python -Wall manage.py test auth_lifecycle.registration.test_login_user_pass_len_vars

See the top of <link to .test__utilities> for more information.
"""
from auth_lifecycle.test__utilities         import assert_pattern_in_string
from django.test              import TestCase
from auth_lifecycle.models    import USERNAME_MIN_LEN, USERNAME_MAX_LEN
from auth_lifecycle.models    import PASSWORD_MIN_LEN, PASSWORD_MAX_LEN
from django.core.urlresolvers import reverse

class UserPassLenVarsTestCase(TestCase):
    """
    Tests to confirm that the username and password min/max length
    variables are correctly passed through from the view, and assigned as
    expected.
    """

    def test_all_config_vars_passed(self):
        """
        Tests to confirm that the username and password min/max length
        variables are correctly passed through from the view, and assigned
        as expected.
        """
        self.client.logout() #Don't really have to, but why not?

        content_str = str(self.client.get(reverse('login')).content)

        assert_pattern_in_string(self,
            r"\bminUserLen += +" + str(USERNAME_MIN_LEN) + ";", content_str)
        assert_pattern_in_string(self,
            r"\bmaxUserLen += +" + str(USERNAME_MAX_LEN) + ";", content_str)
        assert_pattern_in_string(self,
            r"\bminPassLen += +" + str(PASSWORD_MIN_LEN) + ";", content_str)
        assert_pattern_in_string(self,
            r"\bmaxPassLen += +" + str(PASSWORD_MAX_LEN) + ";", content_str)

Output:

/home/myname/django_auth_lifecycle/djauth_venv/lib/python3.4/imp.py:32: PendingDeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  PendingDeprecationWarning)
Creating test database for alias 'default'...
.
----------------------------------------------------------------------
Ran 1 test in 0.039s

OK
Destroying test database for alias 'default'...

Our tests passed.

Give it a try!

Follow these steps to start your server, and try to login with no username and/or password, or one that is too short.

In the next post, we’ll complete our login page functionality with a forgot-my-password link. After that, it’s onto the change your password form, and finally, creating and deleting an account.

[TOC: one, two, three, four, five, six, seven, eight, nine, ten]

…to be continued…

(cue cliffhanger segue music)

Tip for debugging Django templates

Put this anywhere on your page:

<pre>{% filter force_escape %}{% debug %}{% endfilter %}</pre>
{#       //Move THIS open-comment line above the debug line to suppress.
         //Below to activate.
#}

Example output:

DEFAULT_MESSAGE_LEVELS': {'DEBUG': 10,
                            'ERROR': 40,
                            'INFO': 20,
                            'SUCCESS': 25,
                            'WARNING': 30},
 'LANGUAGES': (('af', 'Afrikaans'),
               ('ar', 'Arabic'),
               ('ast', 'Asturian'),
               ('az', 'Azerbaijani'),
               ('bg', 'Bulgarian'),
               ('be', 'Belarusian'),
               ('bn', 'Bengali'),
               ('br', 'Breton'),
               ('bs', 'Bosnian'),
               ('ca', 'Catalan'),
               ('cs', 'Czech'),
               ('cy', 'Welsh'),
               ('da', 'Danish'),
               ('de', 'German'),
               ('el', 'Greek'),
               ('en', 'English'),
               ('en-au', 'Australian English'),
               ('en-gb', 'British English'),
               ('eo', 'Esperanto'),
               ('es', 'Spanish'),
               ('es-ar', 'Argentinian Spanish'),
               ('es-mx', 'Mexican Spanish'),
               ('es-ni', 'Nicaraguan Spanish'),
               ('es-ve', 'Venezuelan Spanish'),
               ('et', 'Estonian'),
               ('eu', 'Basque'),
               ('fa', 'Persian'),
               ('fi', 'Finnish'),
               ('fr', 'French'),
               ('fy', 'Frisian'),
               ('ga', 'Irish'),
               ('gl', 'Galician'),
               ('he', 'Hebrew'),
               ('hi', 'Hindi'),
               ('hr', 'Croatian'),
               ('hu', 'Hungarian'),
               ('ia', 'Interlingua'),
               ('id', 'Indonesian'),
               ('io', 'Ido'),
               ('is', 'Icelandic'),
               ('it', 'Italian'),
               ('ja', 'Japanese'),
               ('ka', 'Georgian'),
               ('kk', 'Kazakh'),
               ('km', 'Khmer'),
               ('kn', 'Kannada'),
               ('ko', 'Korean'),
               ('lb', 'Luxembourgish'),
               ('lt', 'Lithuanian'),
               ('lv', 'Latvian'),
               ('mk', 'Macedonian'),
               ('ml', 'Malayalam'),
               ('mn', 'Mongolian'),
               ('mr', 'Marathi'),
               ('my', 'Burmese'),
               ('nb', 'Norwegian Bokmal'),
               ('ne', 'Nepali'),
               ('nl', 'Dutch'),
               ('nn', 'Norwegian Nynorsk'),
               ('os', 'Ossetic'),
               ('pa', 'Punjabi'),
               ('pl', 'Polish'),
               ('pt', 'Portuguese'),
               ('pt-br', 'Brazilian Portuguese'),
               ('ro', 'Romanian'),
               ('ru', 'Russian'),
               ('sk', 'Slovak'),
               ('sl', 'Slovenian'),
               ('sq', 'Albanian'),
               ('sr', 'Serbian'),
               ('sr-latn', 'Serbian Latin'),
               ('sv', 'Swedish'),
               ('sw', 'Swahili'),
               ('ta', 'Tamil'),
               ('te', 'Telugu'),
               ('th', 'Thai'),
               ('tr', 'Turkish'),
               ('tt', 'Tatar'),
               ('udm', 'Udmurt'),
               ('uk', 'Ukrainian'),
               ('ur', 'Urdu'),
               ('vi', 'Vietnamese'),
               ('zh-cn', 'Simplified Chinese'),
               ('zh-hans', 'Simplified Chinese'),
               ('zh-hant', 'Traditional Chinese'),
               ('zh-tw', 'Traditional Chinese')),
 'LANGUAGE_BIDI': False,
 'LANGUAGE_CODE': 'en-us',
 'MEDIA_URL': '',
 'STATIC_URL': '/static/',
 'TIME_ZONE': 'UTC',
 'csrf_token': <django.utils.functional.lazy..__proxy__ object at 0xb5dc016c>,
 'messages': ,
 'perms': ,
 'user': <SimpleLazyObject: <function AuthenticationMiddleware.process_request.. at 0xb6766df4>>}{'form': ,
 'next': '/auth_lifecycle/',
 'site': ,
 'site_name': '127.0.0.1:8001'}{'False': False, 'None': None, 'True': True}

{'__future__': <module '__future__' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/__future__.py'>,
 '__main__': <module '__main__' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/bin/gunicorn'>,
 '_ast': <module '_ast' (built-in)>,
 '_bisect': <module '_bisect' (built-in)>,
 '_bootlocale': <module '_bootlocale' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/_bootlocale.py'>,
 '_bz2': <module '_bz2' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/_bz2.cpython-34m-i386-linux-gnu.so'>,
 '_codecs': <module '_codecs' (built-in)>,
 '_collections': <module '_collections' (built-in)>,
 '_collections_abc': <module '_collections_abc' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/_collections_abc.py'>,
 '_compat_pickle': <module '_compat_pickle' from '/usr/lib/python3.4/_compat_pickle.py'>,
 '_datetime': <module '_datetime' (built-in)>,
 '_decimal': <module '_decimal' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/_decimal.cpython-34m-i386-linux-gnu.so'>,
 '_frozen_importlib': <module 'importlib._bootstrap' (frozen)>,
 '_functools': <module '_functools' (built-in)>,
 '_hashlib': <module '_hashlib' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/_hashlib.cpython-34m-i386-linux-gnu.so'>,
 '_heapq': <module '_heapq' (built-in)>,
 '_imp': <module '_imp' (built-in)>,
 '_io': <module 'io' (built-in)>,
 '_json': <module '_json' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/_json.cpython-34m-i386-linux-gnu.so'>,
 '_locale': <module '_locale' (built-in)>,
 '_lzma': <module '_lzma' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/_lzma.cpython-34m-i386-linux-gnu.so'>,
 '_markupbase': <module '_markupbase' from '/usr/lib/python3.4/_markupbase.py'>,
 '_opcode': <module '_opcode' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/_opcode.cpython-34m-i386-linux-gnu.so'>,
 '_operator': <module '_operator' (built-in)>,
 '_pickle': <module '_pickle' (built-in)>,
 '_posixsubprocess': <module '_posixsubprocess' (built-in)>,

...

 'psycopg2._json': <module 'psycopg2._json' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/site-packages/psycopg2/_json.py'>,
 'psycopg2._psycopg': <module 'psycopg2._psycopg' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/site-packages/psycopg2/_psycopg.cpython-34m.so'>,
 'psycopg2._range': <module 'psycopg2._range' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/site-packages/psycopg2/_range.py'>,
 'psycopg2.extensions': <module 'psycopg2.extensions' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/site-packages/psycopg2/extensions.py'>,
 'psycopg2.tz': <module 'psycopg2.tz' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/site-packages/psycopg2/tz.py'>,
 'pwd': <module 'pwd' (built-in)>,
 'queue': <module 'queue' from '/usr/lib/python3.4/queue.py'>,
 'quopri': <module 'quopri' from '/usr/lib/python3.4/quopri.py'>,
 'random': <module 'random' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/random.py'>,
 're': <module 're' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/re.py'>,
 'reprlib': <module 'reprlib' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/reprlib.py'>,
 'resource': <module 'resource' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/lib-dynload/resource.cpython-34m-i386-linux-gnu.so'>,
 'select': <module 'select' (built-in)>,
 'selectors': <module 'selectors' from '/usr/lib/python3.4/selectors.py'>,
 'shutil': <module 'shutil' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/shutil.py'>,
 'signal': <module 'signal' (built-in)>,
 'site': <module 'site' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/site.py'>,
 'sitecustomize': <module 'sitecustomize' from '/usr/lib/python3.4/sitecustomize.py'>,
 'socket': <module 'socket' from '/usr/lib/python3.4/socket.py'>,
 'socketserver': <module 'socketserver' from '/usr/lib/python3.4/socketserver.py'>,
 'sre_compile': <module 'sre_compile' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/sre_compile.py'>,
 'sre_constants': <module 'sre_constants' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/sre_constants.py'>,
 'sre_parse': <module 'sre_parse' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/sre_parse.py'>,
 'ssl': <module 'ssl' from '/usr/lib/python3.4/ssl.py'>,
 'stat': <module 'stat' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/stat.py'>,
 'string': <module 'string' from '/usr/lib/python3.4/string.py'>,
 'stringprep': <module 'stringprep' from '/usr/lib/python3.4/stringprep.py'>,
 'struct': <module 'struct' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/struct.py'>,
 'subprocess': <module 'subprocess' from '/usr/lib/python3.4/subprocess.py'>,
 'symbol': <module 'symbol' from '/usr/lib/python3.4/symbol.py'>,
 'sys': <module 'sys' (built-in)>,
 'sysconfig': <module 'sysconfig' from '/usr/lib/python3.4/sysconfig.py'>,
 'tarfile': <module 'tarfile' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/tarfile.py'>,
 'tempfile': <module 'tempfile' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/tempfile.py'>,
 'textwrap': <module 'textwrap' from '/usr/lib/python3.4/textwrap.py'>,
 'threading': <module 'threading' from '/usr/lib/python3.4/threading.py'>,
 'time': <module 'time' (built-in)>,
 'token': <module 'token' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/token.py'>,
 'tokenize': <module 'tokenize' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/tokenize.py'>,
 'traceback': <module 'traceback' from '/usr/lib/python3.4/traceback.py'>,
 'types': <module 'types' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/types.py'>,
 'unicodedata': <module 'unicodedata' (built-in)>,
 'unittest': <module 'unittest' from '/usr/lib/python3.4/unittest/__init__.py'>,
 'unittest.case': <module 'unittest.case' from '/usr/lib/python3.4/unittest/case.py'>,
 'unittest.loader': <module 'unittest.loader' from '/usr/lib/python3.4/unittest/loader.py'>,
 'unittest.main': <module 'unittest.main' from '/usr/lib/python3.4/unittest/main.py'>,
 'unittest.result': <module 'unittest.result' from '/usr/lib/python3.4/unittest/result.py'>,
 'unittest.runner': <module 'unittest.runner' from '/usr/lib/python3.4/unittest/runner.py'>,
 'unittest.signals': <module 'unittest.signals' from '/usr/lib/python3.4/unittest/signals.py'>,
 'unittest.suite': <module 'unittest.suite' from '/usr/lib/python3.4/unittest/suite.py'>,
 'unittest.util': <module 'unittest.util' from '/usr/lib/python3.4/unittest/util.py'>,
 'urllib': <module 'urllib' from '/usr/lib/python3.4/urllib/__init__.py'>,
 'urllib.error': <module 'urllib.error' from '/usr/lib/python3.4/urllib/error.py'>,
 'urllib.parse': <module 'urllib.parse' from '/usr/lib/python3.4/urllib/parse.py'>,
 'urllib.request': <module 'urllib.request' from '/usr/lib/python3.4/urllib/request.py'>,
 'urllib.response': <module 'urllib.response' from '/usr/lib/python3.4/urllib/response.py'>,
 'uu': <module 'uu' from '/usr/lib/python3.4/uu.py'>,
 'warnings': <module 'warnings' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/warnings.py'>,
 'weakref': <module 'weakref' from '/home/jeffy/django_files/djauth_lifecycle_tutorial/part_06/dalt06_venv/lib/python3.4/weakref.py'>,
 'wsgiref': <module 'wsgiref' from '/usr/lib/python3.4/wsgiref/__init__.py'>,
 'wsgiref.handlers': <module 'wsgiref.handlers' from '/usr/lib/python3.4/wsgiref/handlers.py'>,
 'wsgiref.headers': <module 'wsgiref.headers' from '/usr/lib/python3.4/wsgiref/headers.py'>,
 'wsgiref.simple_server': <module 'wsgiref.simple_server' from '/usr/lib/python3.4/wsgiref/simple_server.py'>,
 'wsgiref.util': <module 'wsgiref.util' from '/usr/lib/python3.4/wsgiref/util.py'>,
 'xml': <module 'xml' from '/usr/lib/python3.4/xml/__init__.py'>,
 'xml.dom': <module 'xml.dom' from '/usr/lib/python3.4/xml/dom/__init__.py'>,
 'xml.dom.NodeFilter': <module 'xml.dom.NodeFilter' from '/usr/lib/python3.4/xml/dom/NodeFilter.py'>,
 'xml.dom.domreg': <module 'xml.dom.domreg' from '/usr/lib/python3.4/xml/dom/domreg.py'>,
 'xml.dom.minicompat': <module 'xml.dom.minicompat' from '/usr/lib/python3.4/xml/dom/minicompat.py'>,
 'xml.dom.minidom': <module 'xml.dom.minidom' from '/usr/lib/python3.4/xml/dom/minidom.py'>,
 'xml.dom.xmlbuilder': <module 'xml.dom.xmlbuilder' from '/usr/lib/python3.4/xml/dom/xmlbuilder.py'>,
 'zipfile': <module 'zipfile' from '/usr/lib/python3.4/zipfile.py'>,
 'zipimport': <module 'zipimport' (built-in)>,
 'zlib': <module 'zlib' (built-in)>}

Configuring Nginx with a public static directory (for JavaScript, images, etc.)

This post describes how I configure a public static directory for a Django project, with Nginx (on Ubuntu) as the web server. This is where public, non-secure JavaScript, images, and other documents can be placed, in order to lessen the load on the WSGI Django server. Your server is likely different than mine, so you will have to tailor these steps as necessary.

The Nginx configuration file

This is an Nginx configuration file for a single Django project. You can add in whatever locations-alias blocks you like.

server {
   server_name 104.131.200.120;

   access_log on;

   location /auth_lifecycle/static/ {
       alias /home/myname/django_auth_lifecycle/root/static/;
   }

   #Static images for the admin
   location /static/admin/ {
       alias /home/myname/django_auth_lifecycle/venv/lib/python3.4/site-packages/django/contrib/admin/static/admin/;
   }

   location / {
       proxy_pass http://127.0.0.1:8001;
       proxy_set_header X-Forwarded-Host $server_name;
       proxy_set_header X-Real-IP $remote_addr;
       add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"';
   }
}

The steps I take:

  1. sudo nano /etc/nginx/sites-available/django_project_name
  2. Replace the contents in that file with the above text.
  3. Save and close it with Ctrl+X and then Enter.
  4. Create the directory
        /home/myname/django_auth_lifecycle/root/static/
  5. Restart Nginx: sudo service nginx restart
  6. To confirm your setup, create a file named
        /home/myname/django_auth_lifecycle/root/static/temp.txt
    and put something in it, such as with

        cat x > /home/myname/django_auth_lifecycle/root/static/temp.txt
  7. In your browser, you should be able to see that text at the url
        http-colon-slash-slashmy.website/auth_lifecycle/static/temp.txt
    Delete the file before proceeding.

Since the (non-admin) static location is set to /auth_lifecycle/static/, this must be mirrored in Django, by setting this same value to the STATIC_URL variable in settings.py:

STATIC_URL = "/auth_lifecycle/static/"

An aside: Since this configuration really belongs to this Django project, I recommend storing the above configuration text in its entirety, in settings.py, (in a multi-line comment: """..."""), immediately above the STATIC_URL variable.

Referencing static files in a Django template

Although you could use

<link href="{{ STATIC_URL }}stylesheets/tabs.css" ...>

You should instead use the “static” template tag:

{% load staticfiles %}
{% static 'stylesheets/tabs.css' %}