Solaris 11: Setting up Sendmail / SASL to handle SMTP AUTH

I thought I would finally get SMTP AUTH working on my Solaris 11 servers, so that I could then setup my phone to send e-mail correctly. Solaris 11 can't do it with the standard software.

Build Cyrus SASL

• Install the packages which we requite and download Cyrus SASL Library
  
  # sudo pkg install pkg:/developer/build/make system/header developer/gcc-3 text/gnu-grep database/berkeleydb-5
  
  # echo $PATH
  /usr/sbin:/usr/bin
  # export PATH=$PATH:/usr/gnu/bin:/usr/sfw/bin
  
  
• At this time it will not compile, so alter the following lines.
  # gzip -dc cyrus-sasl-2.1.25 | tar xf -
  # cd cyrus-sasl-2.1.25
  
  # vi ./lib/saslutil.c ./plugins/kerberos4.c ./plugins/digestmd5.c utils/Makefile.am
  ./lib/saslutil.c
  85c85
  extern int gethostname(char *, int);
  change to:
  extern int gethostname(char *, unsigned int);
  
  ./plugins/kerberos4.c
  113c113
  extern int gethostname(char *, int);
  change to:
  extern int gethostname(char *, unsigned int);
  
  ./plugins/digestmd5.c
  106c106
  extern int gethostname(char *, int);
  change to:
  extern int gethostname(char *, unsigned int);
  
  ./utils/Makefile.am
  59a60,62
  add the following lines:
  saslpasswd2_LDFLAGS = -rpath $(libdir)
  dbconverter_2_LDFLAGS = -rpath $(libdir)
  pluginviewer_LDFLAGS = -rpath $(libdir)
  
  
• Now we are ready for building.
  # ./configure --enable-cram --enable-digest --enable-plain \
            --enable-login --disable-krb4 --disable-anon \
            --disable-gssapi --with-saslauthd=/var/run/saslauthd
  # make
  # sudo make install
  
  
• Add the following links:
  # sudo ln -s /usr/local/lib/sasl2 /usr/lib/sasl2
  # sudo ln -s /usr/local/lib/libsasl2.so.2.0.25 /usr/lib/libsasl2.so.2

Build Sendmail

• Download sendmail  8.14.5
  
• Stop existing sendmail processes.
  # sudo svcadm disable svc:/network/smtp:sendmail
  # sudo svcadm disable svc:/network/sendmail-client:default
  
  # gzip -dc sendmail.8.14.5.tar.gz | tar xf -
  # cd sendmail-8.14.5
  
  
• Set up sendmail FEATURES (SSL, SASL, DB)
  # cat > devtools/Site/site.config.m4
  dnl ### Changes to disable the default NIS support
  APPENDDEF(`confENVDEF', `-UNIS')
  
  dnl ### Changes for STARTTLS support
  APPENDDEF(`confENVDEF',`-DSTARTTLS')
  APPENDDEF(`confLIBS', `-lssl -lcrypto')
  APPENDDEF(`confLIBDIRS', `-L/usr/local/lib -L/usr/lib -R/usr/lib')
  APPENDDEF(`confINCDIRS', `-I/usr/include/openssl')
  
  dnl ### SASL support
  APPENDDEF(`confENVDEF', `-DSASL=2')
  APPENDDEF(`conf_sendmail_LIBS', `-lsasl2')
  APPENDDEF(`confINCDIRS', `-I/usr/local/sasl -I/usr/include')
  
  dnl ### Berkley DB support
  APPENDDEF(`confENVDEF', `-DNEWDB')
  APPENDDEF(`confINCDIRS', `-I/usr/include')
  APPENDDEF(`confLIBDIRS', `-L/usr/lib')
  
  dnl ### TCP Wrapper
  APPENDDEF(`confENVDEF', `-DTCPWRAPPERS')
  APPENDDEF(`conf_sendmail_LIBS', `-lwrap')
  EOF
  #

• Build Sendmail
  # ./Build -c
  /* It should build but ignore the gtroff:error at the end */
  
  # sudo mv /usr/lib/sendmail /usr/lib/sendmail_orig
  # sudo cp obj.SunOS.5.11.i86pc/sendmail/sendmail /usr/lib/sendmail
  # sudo sudo chgrp smmsp /usr/lib/sendmail
  # sudo sudo chmod g+s,u-ws /usr/lib/sendmail
  
  
• Now a Vendor and SASL support to sendmail.cf file (I hope you know what you are doing here)
• Edit sendmil.mc add add the extra lines:
  define(`VENDOR_NAME', `Berkeley')
  define(`confAUTH_OPTIONS', `A')dnl
  TRUST_AUTH_MECH(`EXTERNAL DIGEST-MD5 CRAM-MD5 LOGIN PLAIN')dnl
  define(`confAUTH_MECHANISMS', `DIGEST-MD5 CRAM-MD5 LOGIN PLAIN')dnl
  
  # cd /etc/mail/cf/cf
  # cat sendmail.mc
  divert(0)dnl
  VERSIONID(`sendmail.mc (Sun)')
  define(`VENDOR_NAME', `Berkeley')
  OSTYPE(`solaris11')dnl
  DOMAIN(`solaris-generic')dnl
  define(`confAUTH_OPTIONS', `A')dnl
  TRUST_AUTH_MECH(`EXTERNAL DIGEST-MD5 CRAM-MD5 LOGIN PLAIN')dnl
  define(`confAUTH_MECHANISMS', `DIGEST-MD5 CRAM-MD5 LOGIN PLAIN')dnl define(`confFALLBACK_SMARTHOST', `mailhost$?m.$m$.')dnl
  MAILER(`local')dnl
  MAILER(`smtp')dnl
  
  LOCAL_NET_CONFIG
  R$* < @ $* .$m. > $* $#esmtp $@ $2.$m $: $1 < @ $2.$m. > $3
  #
  # sudo make sendmail.cf
  # sudo cp sendmail.cf ../../sendmail.cf
  
  # sudo /usr/lib/sendmail -bt -d0 < /dev/null
  Version 8.14.5
  Compiled with: DNSMAP LOG MAP_REGEX MATCHGECOS MILTER MIME7TO8 MIME8TO7
      NAMED_BIND NDBM NETINET NETINET6 NETUNIX NEWDB PIPELINING
      SASLv2 SCANF STARTTLS TCPWRAPPERS USERDB XDEBUG

Setup sasl authentication server

• Now lets set up SASL plugin via a Sendmail.conf
  # sudo cat > /usr/local/lib/sasl2/Sendmail.conf
  pwcheck_method: saslauthd
  EOF
  #
  
  
• For authentication to work the saslauthd has to be started, so you can start in as a one off process (good for debugging) and at boot time.
  # sudo /usr/local/sbin/saslauthd -n 1 -V -d -a pam
  saslauthd[398] :main : num_procs : 1
  saslauthd[398] :main : mech_option: NULL
  saslauthd[398] :main : run_path : /var/run/saslauthd
  saslauthd[398] :main : auth_mech : pam
  saslauthd[398] :ipc_init : using accept lock file: /var/run/saslauthd/mux.accept
  saslauthd[398] :detach_tty : master pid is: 0
  saslauthd[398] :ipc_init : listening on socket: /var/run/saslauthd/mux
  saslauthd[398] :main : using process model
  saslauthd[398] :get_accept_lock : acquired accept lock
  
  
• To set it up ant boot time we should set up a services, but for now we will stick to init.d files.
  cat > /etc/init.d/saslauthd
  #!/usr/sbin/sh
  #
  NAME=saslauthd
  DAEMON="/usr/local/sbin/${NAME}"
  DESC="SASL Authentication Daemon"
  
  # -a Selects the authentication mechanism to use.
  # -n Number of worker processes to create.
  # -V Enable verbose logging
  # -d Debugging (don't detach from tty, implies -V)
  
  case "$1" in
  'start')
  ${DAEMON} -n 1 -V -d -a pam
  ;;
  
  'stop')
  pgrep saslauthd
  ;;
  
  *)
  echo "Usage: $0 { start | stop }"
  exit 1
  ;;
  esac
  EOF
  #
  # sudo ln init.d/saslauthd rc0.d/K38saslauthd
  # sudo ln init.d/saslauthd rc1.d/K38saslauthd
  # sudo ln init.d/saslauthd rc2.d/S82saslauthd
  # sudo ln init.d/saslauthd rcS.d/K38saslauthd
  
  
  
• Time for some testing, via first the SASL daemon.
  # cyrus-sasl-2.1.25/saslauthd/testsaslauthd -u RealUser -p MyPassword
  0: OK "Success."
  
  
• We need the encrypted username and password to test sendmail:
  # perl -MMIME::Base64 -e 'print encode_base64("\000MyUser\000MyPassword")'
  AE15VXNlcgBNeVBhc3N3b3Jk
  
  
• Test the sendmail part:
  # sudo /usr/lib/sendmail -bv  -O LogLevel=14 -bs -Am
  220 mailer5.dcs.bbk.ac.uk ESMTP Sendmail 8.14.5/8.14.5; Tue, 31 Jan 2012 09:52:24 GMT
  EHLO localhost
  250-mailer5.dcs.bbk.ac.uk Hello root@localhost, pleased to meet you
  250-ENHANCEDSTATUSCODES
  250-PIPELINING
  250-EXPN
  250-VERB
  250-8BITMIME
  250-SIZE
  250-DSN
  250-ETRN
  250-AUTH DIGEST-MD5 CRAM-MD5 LOGIN PLAIN
  250-DELIVERBY
  250 HELP
  AUTH PLAIN AE15VXNlcgBNeVBhc3N3b3Jk
  235 2.0.0 OK Authenticated
  quit
  221 2.0.0 mailer5.dcs.bbk.ac.uk closing connection
  
  
• That is it.