\documentclass{beamer}

% You can also use a 16:9 aspect ratio:
%\documentclass[aspectratio=169]{beamer}
\usetheme{TACC16}

% It's possible to move the footer to the right:
%\usetheme[rightfooter]{TACC16}

\begin{document}
\title[Lmod]{Lmod: A Modern Environment Module System\\
             XALT: Understanding HPC Usage via Job Level Collection}

% page 1
\frame{\titlepage} 

% page 2
\begin{frame}{Outline}
  \begin{itemize}
    \item What are Environment Modules?
    \item What is Lmod?
    \item Can Lmod help manage your site?
    \item Advanced Topics
    \item Where to go for help.
  \end{itemize}
\end{frame}

% page 3
\begin{frame}{What are Modules?}
  \begin{itemize}
    \item Modules help user load the SW packages they need: MPI, Boost, ...
    \item Modules can be unloaded.
    \item Modulefiles are in one file not one for each
      shell. (e.g. compiler init scripts)
    \item It is one of the main ways we communicate with our users.
    \item The main function of Lmod is to change the user environment
    \item It modifies PATH, LD\_LIBRARY\_PATH, ...
    \item It sets other environment variables and defines aliases
  \end{itemize}
\end{frame}

% page 4
\begin{frame}{What is Lmod?}
  \begin{itemize}
    \item A modern replacement for a tried and true concept.
    \item The guiding principal: ``Make life easier without getting in
      the way.''
    \item Reads both TCL and Lua modulefiles
  \end{itemize}
\end{frame}

% page 5
\begin{frame}{Fundamental Issues}
  \begin{itemize}
    \item Software Packages are created and updated all the time.
    \item Some Users need new versions for new features and bug fixes.
    \item Other Users need older versions for stability and continuity.
    \item No system can support all versions of all packages.
    \item User programs using pre-built C++ \& Fortran libraries must link with the same compiler.
    \item Similarly, MPI Applications must build and link with same
      MPI/Compiler pairing when using pre-built MPI libraries.
  \end{itemize}
\end{frame}


% page 6
\begin{frame}[fragile]
    \frametitle{Example of Lmod: Environment Modules (I)}
    {\tiny
\begin{semiverbatim}
{\color{blue}\$ module list}
Currently Loaded Modules:
  1) StdEnv  2) gcc/4.5  3) mpich2/1.4  4) petsc/3.1
{\color{blue}\$ module unload gcc}
Inactive Modules:
  1) mpich2  2) petsc
{\color{blue}\$ module list}
Currently Loaded Modules:
  1) StdEnv
Inactive Modules:
  1) mpich2  2) petsc
{\color{blue}\$ module load intel}
Activating Modules:
  1) mpich2  2) petsc
{\color{blue}\$ module swap intel gcc}
Due to MODULEPATH changes the follow modules have been reloaded:
  1) mpich2  2) petsc
{\color{blue}\$ module load gcc}
Due to MODULEPATH changes the follow modules have been reloaded:
  1) mpich2  2) petsc
\end{semiverbatim}
    }
\end{frame}

% page 7
\begin{frame}[fragile]
    \frametitle{Example of Lmod: Environment Modules (II)}
    {\tiny
\begin{semiverbatim}
\$ {\color{blue} module avail}
------------------ /opt/apps/modulefiles/MPI/intel/12.0/mpich2/1.4 ------------------
  petsc/3.1 (D)    petsc/3.1-debug    pmetis/4.0    tau/2.20.3

------------------- /opt/apps/modulefiles/Compiler/intel/12.0 -----------------------
  boost/1.45.0        gotoblas2/1.13      openmpi/1.4.3
  boost/1.46.0        mpich2/1.3.2        openmpi/1.5.1
  boost/1.46.1 (D)    mpich2/1.4    (D)   openmpi/1.5.3   (D)

-------------------------- /opt/apps/modulefiles/Core -------------------------------
  StdEnv               intel/11.1         papi/4.1.4
  admin/admin-1.0      intel/12.0  (D)    scite/2.28
  ddt/ddt              lmod/lmod          tex/2010
  dmalloc/dmalloc      local/local (D)    unix/unix    (D)
  fdepend/1.2          mkl/mkl            visit/visit
  gcc/4.4              noweb/2.11b
  gcc/4.5        (D)
\end{semiverbatim}
    }
\end{frame}


% page 8
\begin{frame}{Why does Lmod work at all?}
  \begin{itemize}
    \item The environment is inherited from the parent process
    \item Changes in a child process \emph{DOES NOT} affect the
      parent's environment
    \item So how could Lmod work at all?
  \end{itemize}
\end{frame}

% page 9
\begin{frame}{The trick is:}
  \begin{itemize}
    \item The \texttt{lmod} program generates text.
    \item The module command eval's that text.
    \item \texttt{module() \{eval \$(\$LMOD\_CMD bash "\$@";)\}}
    \item \texttt{stdout} is evaluated
    \item \texttt{stderr} passes through
  \end{itemize}
\end{frame}


% page 10
\begin{frame}{Can tools like Lmod improve the user experience?}
  \begin{itemize}
    \item Sites provide packages: applications and libraries
    \item Users can pick which packages and version to suit their needs
    \item But what we are really after is to cut down on tickets!
    \item Or simply make your resources easier for your users
  \end{itemize}
\end{frame}

% page 11
\begin{frame}{Lmod examples}
  \begin{itemize}
    \item Lmod was first released in 2009
    \item It is the only module system used at TACC since 2010
    \item The following are some examples of how Lmod can help
  \end{itemize}
\end{frame}

% page 12
\begin{frame}{Can Lmod help with the /usr/local/bin problem?}
  \begin{itemize}
    \item Suppose your startup files put /usr/local/bin in PATH
    \item And suppose module BAR also adds /usr/local/bin to PATH
    \item Currently Loading then unloading BAR will remove
      /usr/local/bin from PATH. 
    \item Site can configure Lmod to support duplicate paths
    \item Or coming soon Lmod will support reference counting!
  \end{itemize}
\end{frame}

% page 13
\begin{frame}{Can Lmod prevent users from mixing modules they shouldn't?}
  \begin{itemize}
      \item Same Name modules:
      \begin{itemize}
        \item Things can get confusing when users load two gcc modules
        \item Normally, Lmod will unload old gcc, then load new gcc
        \item Optionally, sites can auto-conflict with themselves.
      \end{itemize}
    \item Loading two compilers or MPI Stack:
      \begin{itemize}
        \item It is a rare user who needs to load two different
          compilers or two MPI stacks
        \item GCC and Intel are a special case
        \item Sites can add family("compiler") to compiler modules
        \item This will autoswap one compiler for another!
        \item Similarly for MPI modules.
      \end{itemize}
  \end{itemize}
\end{frame}

% page 14
\begin{frame}{How to manage software: New or Old}
  \begin{itemize}
    \item How can you test new/experimental software?
    \item Suppose your site keeps SW for the life of machine?
    \item How do you encourage usage of newer SW w/o breaking old job
      scripts?
    \item Lmod now supports hiding regular modules from avail and
      spider.
    \item Hidden modules can still be loaded.
    \item Modules can be explicitly marked as hidden
    \item Or you can use the isVisible hook
    \item Both sites and users can hide modules
  \end{itemize}
\end{frame}

% page 15
\begin{frame}{Can Lmod help with deprecating packages?}
  \begin{itemize}
    \item Suppose your site keeps a limited number of versions (say 3
      or less)
    \item How to you decide which package to keep or remove?
    \item Lmod support optional tracking of what packages are loaded
      by whom.
    \item You can send targeted email to those users about
      deprecation based on tracking.
    \item Independent of tracking: nag messaging
    \item Do not need to change modulefile!
    \item Users get a message when they load a deprecated module. 
  \end{itemize}
\end{frame}

% page 16
\begin{frame}{Can Lmod help a site that does not want default modules?}
  \begin{itemize}
    \item Suppose your site produces weather forecasts or processes
      satellite images.
    \item No one set of compilers etc will satify your needs.
    \item Site can set LMOD\_EXACT\_MATCH$=$yes $\Rightarrow$ There are no defaults
    \item Users \emph{MUST} specify name and version!
  \end{itemize}
\end{frame}

% page 17
\begin{frame}{Can users have their own default list of modules?}
  \begin{itemize}
    \item It is common to provide a default list of modules
    \item However some users will want their own modules at startup
    \item Users can add module commands to \textasciitilde/.bashrc etc
    \item But this is tricky to get right.
    \item Lmod supports default module collections
    \item In fact, users can have as many named collections as they like.
  \end{itemize}
\end{frame}


% page 18
\begin{frame}{Can Lmod deal with shared home filesystem?}
  \begin{itemize}
    \item Suppose your site shares the home filesystem across two or
      more clusters
    \item These clusters have different modules.
    \item Site can set \$LMOD\_SYSTEM\_NAME uniquely on each cluster
    \item This way user's collection (and personal caches) will be
      unique
  \end{itemize}
\end{frame}

% page 19
\begin{frame}{Can users easily grep the output from Lmod?}
  \begin{itemize}
    \item Lmod sends messages to \texttt{stderr} by default
    \item Lmod can redirect the output to \texttt{stdout} by setting
      \$LMOD\_REDIRECT=yes
    \item This works for bash, zsh
    \item It doesn't work for csh/tcsh due to the way eval works there
    \item Setting \$LMOD\_REDIRECT=yes means you lose the pager
    \item I do this instead: \$ module --raw --redirect show impi |
      grep tmi
  \end{itemize}
\end{frame}

%page 20
\begin{frame}[fragile]
    \frametitle{Can a site control the output of module avail?}
  {\tiny
    \begin{semiverbatim}
 $ module av
---- Packages compiled w/ Intel compilers -----------
   ABINIT/8.2.2           ParaView/5.3.0-OSMesa
   ARPACK-NG/3.4.0        PyMOL/1.8.6-Python-2.7.13
  [...]

------------ MPI available for Intel compilers ------
   IntelMPI/2017.2.174    ParaStationMPI/5.1.9-1 (D)
   [...]
------ Packages compiled with Intel compilers -------
   Eigen/3.3.3            Libxc/2.2.3 
   [...]
--------------- Core packages -----------------------
   Advisor/2017\_update2      PostgreSQL/9.6.2
   [...]

----------------- Compilers -------------------------
   GCC/5.4.0                Intel/17.2-GCC-5.4 (D)
   Intel/16.4-GCC-5.4       PGI/17.3-GCC-5.4

---------------- Recommended defaults ---------------
   defaults/CPU (D)         defaults/GPU (g)
\end{semiverbatim}
%$
}
\end{frame}

% page 21
\begin{frame}{Can Lmod work with Localization and Site Messages?}
  \begin{itemize}
    \item Starting Lmod 7.1+ Lmod provides the possibility of Language
      Translations: ES, FR, DE, and ZH
    \item Sites can also provide tailored message to suit their needs
  \end{itemize}
\end{frame}

% page 22
\begin{frame}{Can Lmod help with software web pages?}
  \begin{itemize}
    \item Many sites want to provide web pages that list the SW
      they provide.
    \item Lmod provides a tool to generate a JSON or XML list of all
      system modules.
    \item You'll have to write something to ingest the JSON or XML
  \end{itemize}
\end{frame}

% page 23
\begin{frame}{Can Lmod help with compiler and/or MPI/compiler
      dependent modules?}
  \begin{itemize}
    \item Sites can chose a Flat or Hierarchical Naming Scheme
    \item PETSc: A parallel iterative solver package:
      \begin{itemize}
        \item Compilers: GCC 6.3, Intel 17.0
        \item MPI Implementations: MVAPICH2 2.1, IMPI 17.0
        \item MPI Solver package: PETSc 4.1
        \item 4 versions of PETSc: 2 Compilers $\times$ 2 MPI
      \end{itemize}

    \item Flat layout for PETSc
      \begin{enumerate}
        \item \texttt{PETSc/4.1-mvapich2-2.1-gcc-6.3}
        \item \texttt{PETSc/4.1-mvapich2-2.1-intel-17.0}
        \item \texttt{PETSc/4.1-impi-17.0-gcc-6.3}
        \item \texttt{PETSc/4.1-impi-17.0-intel-17.0}
      \end{enumerate}
  \end{itemize}
\end{frame}

% page 24
\begin{frame}{Problems w/ Flat naming scheme}
  \begin{itemize}
    \item Users have to load modules:
      \begin{itemize}
        \item ``intel/17.0''
        \item ``mvapich2/2.1-intel-17.0''
        \item ``PETSc/4.1-mvapich2-2.1-intel-17.0''
        \item Changing compilers means unloading all three modules
        \item Reloading new compiler, MPI, PETSc modules.
        \item Not loading correct modules $\Rightarrow$ Mysterious Failures!
        \item Onus of package compatibility on users!
        \item Or extremely complicated modulefiles!
        \item Tools like EasyBuild or Spack can help here.
      \end{itemize}
  \end{itemize}
\end{frame}

%page 25
\begin{frame}{Hierarchical Naming Schemes}
  \begin{itemize}
    \item Store modules under one tree: \texttt{/opt/apps/modulefiles}
    \item One strategy is to use sub-directories:
      \begin{itemize}
        \item Core: Regular packages: apps, compilers, git
        \item Compiler: Packages that depend on compiler: boost, MPI
        \item MPI: Packages that depend on MPI/Compiler: PETSc, FFTW3
      \end{itemize}
  \end{itemize}
\end{frame}

%page 26
\begin{frame}{Loading the correct module}
  \begin{itemize}
    \item User loads ``\texttt{intel/17.0}'' module
    \item Can only see/load compiler dependent packages that are built with
      intel 17.0 compiler.
    \item Can not see/load package built with other versions or other compilers.
    \item Similar loading ``\texttt{mvapich2/2.1}'' module.
    \item Users can only load package that are built w/ intel 17.0 and
      mvapich2 2.1 and no others.
  \end{itemize}
\end{frame}

% page 27
\begin{frame}{Lmod works with both flat or hierarchy layouts}
  \begin{itemize}
    \item Sites can chose either kind of layout.  
    \item Lmod offers many advantages with either layout
    \item An Lmod site sys-admin transitioned his users by leaving the
      old system active
    \item A new hierarchy was available where all new SW was installed.
    \item Users can transition if/when they like.
  \end{itemize}
\end{frame}

%page 28
\begin{frame}{Bash Issues}
  \begin{itemize}
    \item Bash Startup is typically ``broken'' for non-login interactive shells
    \item Redhat, Centos, MacOS typically don't source /etc/bashrc on interactive shells
    \item MPI jobs start an interactive shell.
  \end{itemize}
\end{frame}

%page 29
\begin{frame}{Bash Issues (II)}
  \begin{itemize}
    \item Want module command to work in all shells.
    \item Want stacksize unlimited for MPI jobs
    \item We patched bash to force it to source /etc/tacc/bashrc
  \end{itemize}
\end{frame}

%page 30
\begin{frame}{Bash Repair Choices}
  \begin{itemize}
    \item Switch users to Z shell?
    \item patch bash (see Lmod docs)
    \item Expect all users to source /etc/bashrc in \textasciitilde/.bashrc
    \item Expect all users to start jobs with \#!/bin/bash -l
  \end{itemize}
\end{frame}

%page 31
\begin{frame}{Debugging Lmod}
  \begin{itemize}
    \item \texttt{module --config} : reports Lmod configuration
    \item \texttt{module -D load foo $>$ load.log}
  \end{itemize}
\end{frame}

%page 32
\begin{frame}{Tracing Lmod}
  \begin{itemize}
    \item A new feature of Lmod 7.4.4+
    \item module -T ...
    \item export LMOD\_TRACING=yes
    \item Can trace loads and how restores work.
  \end{itemize}
\end{frame}

% page 33
\begin{frame}{Lmod Features}
  \begin{itemize}
    \item Reads both TCL and Lua modulefiles (mixing works fine)
    \item One name rule.
    \item Support a Software Hierarchy
    \item Fast \texttt{module avail} via optional spider cache 
    \item Properties (gpu, mic)
    \item Semantic Versioning:  5.6 is older than 5.10
    \item family(``compiler''), family(``MPI'') support
    \item Optional Tracking: What modules are used?
    \item Many other features: ml, collections, hooks, nag, ...
  \end{itemize}
\end{frame}

%page 34
\begin{frame}{Conclusions: Lmod}
  \begin{itemize}
    \item Latest version: https://github.com:TACC/Lmod.git
    \item Stable version: http://lmod.sf.net
    \item Documentation:  http://lmod.readthedocs.org
    \item Shell Startup Debug: http://shellstartupdebug.sf.net
    \item Mailing List:   lmod-users@lists.sourceforge.net.
    \item Join here: https://lists.sourceforge.net/lists/listinfo/lmod-users
  \end{itemize}
\end{frame}

\begin{frame}{XALT: What runs on the system}
  \begin{itemize}
    \item A U.S. NSF Funded project: PI: Mark Fahey and Robert McLay
    \item A Census of what programs and libraries are run
    \item Running at TACC, NICS, U. Florida, KAUST, ...
    \item Integrates with TACC-Stats.
  \end{itemize}
\end{frame}

\begin{frame}{Design Goals}
  \begin{itemize}
    \item Be extremely light-weight
    \item Provide provenance data: How?
    \item How many use a library or application?
    \item Collect Data into a Database for analysis.
  \end{itemize}
\end{frame}

\begin{frame}{Design: Linker}
  \begin{itemize}
    \item XALT wraps the linker to enable tracking of exec's
    \item The linker (ld) wrapper intercepts the user link line.
    \item Generate assembly code: key-value pairs
    \item Capture tracemap output from ld
    \item Transmit collected data in *.json format
  \end{itemize}
\end{frame}

\begin{frame}{Design: Launcher}
  \begin{itemize}
    \item XALT 1 used to require a wrapper for aprun, mpirun, etc
    \item XALT 2 no longer needs to
    \item Hooray! Correct wrappers were a nightmare!
  \end{itemize}
\end{frame}

\begin{frame}{Design: Transmission to DB}
  \begin{itemize}
    \item File: collect nightly
    \item Syslog: Use Syslog filtering (or ELK)
    \item Direct to DB.
    \item Future: RabbitMQ
  \end{itemize}
\end{frame}

\begin{frame}{Lmod to XALT connection}
  \begin{itemize}
    \item Lmod spider walks entire module tree.
    \item Can build a reverse map from paths to modules
    \item Can map program \& libraries to modules.
    \item /opt/apps/i15/mv2\_2\_1/phdf5/1.8.14/lib/libhdf5.so.9
      $\Rightarrow$ phdf5/1.8.14(intel/15.02:mvapich2/2.1)
    \item Also helps with function tracking.
    \item Tmod Sites can still use Lmod to build the reverse map.
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
    \frametitle{Database Changes (I)}
  \begin{itemize}
      \item Tables sizes in XALT:
 {\small
    \begin{semiverbatim}
+------------------+------------+
| Table            | Size in MB |
+------------------+------------+
| join\_run\_env     |  199603.00 |
| join\_run\_object  |    9388.00 |
| join\_link\_object |    5013.00 |
| xalt\_run         |    4613.00 |
| xalt\_object      |    4175.00 |
| xalt\_link        |     814.00 |
+------------------+------------+
    \end{semiverbatim}
}
    \item \texttt{join\_run\_env} has 2.1 billion rows
  \end{itemize}
\end{frame}

\begin{frame}{Database Changes (II)}
  \begin{itemize}
    \item Environment variables are important.
    \item But mainly for reproducing results
    \item Chose a few for SQL tests.
  \end{itemize}
\end{frame}

\begin{frame}{Database Changes (III): New Design}
  \begin{itemize}
    \item Store complete env $\Rightarrow$ compressed json blob
    \item Filter Env's with Accept Test followed by Reject Test
    \item Instead of 250 vars per job $\Rightarrow$ 20 to 30.
    \item The Filter is site controllable!
  \end{itemize}
\end{frame}

\begin{frame}{Database Changes (IV): New Design}
  \begin{itemize}
    \item The ``join'' tables are large 
    \item Partition ``join'' tables by dates or index
    \item Precompute views nightly.
  \end{itemize}
\end{frame}

\begin{frame}{Protecting XALT (I): UTF8 Characters}
  \begin{itemize}
    \item Linux supports UTF8 Characters in file names, env. vars.
    \item Python supports UTF8 if you know what you are doing.
    \item Switch XALT to use prepared statements
    \item Where \texttt{query="INSERT INTO table VALUE(?,?)"}
    \item This prevent SQL injection: ``johnny drop tables;''
    \item Also supports UTF8 characters.
  \end{itemize}
\end{frame}

\begin{frame}{Protecting XALT (II): Python to C++}
    \begin{itemize}
      \item Difficult to protect Python from users in every case
      \item Solution: LD\_LIBRARY\_PATH="@ld\_lib\_path@"
        PATH=/usr/bin:/bin C++-exec ...
      \item Everything that depends on PATH must be hard coded
      \item basename $\Rightarrow$ /bin/basename
      \item Unique install for each operating system.
      \item Certain programs aren't in the same place: \texttt{basename}
    \end{itemize}
\end{frame}

\begin{frame}{Using XALT Data}
  \begin{itemize}
    \item Targeted Outreach: Who will be affected
    \item Largemem Queue Overuse
    \item XALT and TACC-Stats
  \end{itemize}
\end{frame}

\begin{frame}{Tracking Non-mpi jobs (I)}
  \begin{itemize}
    \item Originally we tracked only MPI Jobs
    \item By hijacking mpirun etc.
    \item Now we can use ELF binary format to track jobs
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
    \frametitle{ELF Binary Format Trick}
 {\small
    \begin{semiverbatim}
void myinit(int argc, char **argv)
\{
  /* ... */
\}
void myfini()
\{
  /* ... */
\}
  __attribute__((section(".init_array")))
       typeof(myinit) *__init = myinit;
  __attribute__((section(".fini_array")))
       typeof(myfini) *__fini = myfini;
    \end{semiverbatim}
}
\end{frame}

\begin{frame}{Using the ELF Binary Format Trick}
  \begin{itemize}
    \item This C code is compiled and linked in through the hijacked linker
    \item It can also be used with \texttt{LD\_PRELOAD}
    \item We are using both...
  \end{itemize}
\end{frame}

\begin{frame}{Downsides}
  \begin{itemize}
    \item Currently, we only track task 0 jobs.
    \item MPMD programs will only record the Task 0 job.
    \item We also lose the ability to capture return exit status
  \end{itemize}
\end{frame}

\begin{frame}{Challenges (I)}
  \begin{itemize}
    \item Do not want to track mv, cp, etc
    \item Only want to track some executables on compute nodes
    \item Do not want to get overwhelmed by the data. 
  \end{itemize}
\end{frame}

\begin{frame}{Answers}
  \begin{itemize}
    \item XALT Tracking only when told to
    \item Compute node only by host name filtering
    \item Executable Filter based on Path
    \item Protection against closing stderr before fini.
    \item Site configurable!
  \end{itemize}
\end{frame}

\begin{frame}{Path Filtering}
  \begin{itemize}
    \item Accept test, following an Ignore Test,
    \item Two files containing regex patterns, converted to code.
    \item Accept List Tests: Track /usr/bin/ddt, /bin/tar
    \item Ignore List Tests: /usr/bin, /bin, /sbin, ...
  \end{itemize}
\end{frame}

\begin{frame}{Using XALT 2}
  \begin{itemize}
    \item A great deal of hardening 
    \item Been running XALT 2 for 4 months with only 1 tckt.
  \end{itemize}
\end{frame}

\begin{frame}{Speeding up XALT 2}
  \begin{itemize}
    \item XALT 2 generates 2 json records: at start and end
    \item Want to minimize measurement: Launcher jobs
    \item The most expensive operation is sha1sum of the shared libs
    \item Used to system sha1sum call in serial
    \item Now up-to 16 threads calls directly
    \item Tests show 1 sec first time 0.04 second time on Lustre.
  \end{itemize}
\end{frame}


\begin{frame}{XALT Demo}
  \begin{itemize}
    \item Show modules hierarchy
    \item ml --raw show xalt
    \item Show debugging output
    \item type -a {ld,mpirun}
    \item Build programs
    \item Run tests
    \item Run utf8 program
    \item Show database results
  \end{itemize}
\end{frame}

\begin{frame}{Conclusion}
  \begin{itemize}
    \item Lmod:
      \begin{itemize}
        \item Source: github.com/TACC/lmod.git, lmod.sf.net
        \item Documentation: lmod.readthedocs.org
      \end{itemize}
    \item XALT:
      \begin{itemize}
        \item Source: github.com/Fahey-McLay/xalt.git, xalt.sf.net
        \item Documentation: doc/*.pdf, xalt.readthedocs.org
      \end{itemize}
  \end{itemize}
\end{frame}

\end{document}
