#!/usr/bin/perl
# Originally written 15 February 1999 by Jim Lippard as short hack script.
# Rewritten 25 February 1999 by Jim Lippard to use config file and be
#    a bit fancier.  Early a.m. 26 February: changed some error messages,
#    fixed uninitialized variable problem in read_size_file.
# Modified 11 August 1999 by Jim Lippard to support cyclog-format logs.
#    A cyclog is a directory containing files named with timestamps.
#    For cyclogs, we store the time logs were last examined as well as
#    the size of the last log file examined.  We can only detect a
#    reduction in size (editing) on the last log file examined.
#    cyclog is part of Daniel Bernstein's daemontools package, and may
#    be found at ftp://koobera.math.uic.edu/www/daemontools.html
#    This script now requires Time::HiRes from CPAN.
# Modified 26 August 1999 by Jim Lippard to support multilog-format
#    logs.  This is Bernstein's new replacement for cyclog, the format
#    is very similar.  The only real change is when converting the
#    timestamps.  Also modified to sort files within cyclogs/multilogs.
# Modified 23 December 1999 by Jim Lippard to change name on email to
#    "Reporter" and use hostname minus domain name in subject line.
# Modified 23 April 2002 by Jim Lippard to correctly parse regexps which
#    contain colons.
# Modified 12 February 2003 by Jim Lippard to allow the use of a single
#    reportnew.conf file for multiple hosts in a backwards-compatible
#    way by adding optional begin-host: hostname and end-host: hostname
#    fields.  If master_notify appears outside of any begin-host/end-host
#    blocks, it is the master_notify for all hosts.
# Modified 28 June 2003 by Jim Lippard--there appears to be a bug where
#    sometimes notifications are sent when the value in the notify hash
#    is undefined.  A workaround has been put in place to use the master_notify
#    address when an undefined value is sent to the send_notify sub.
# Modified 12 January 2009 by Jim Lippard to convert djbdns log IP addresses.
# Modified 11 February 2011 by Jim Lippard to change size for warning about
#    logfile turning over.
# Modified 3 December 2011 by Jim Lippard to catch an error condition
#    that leads to an unitialized value for $notify_list in the "To"
#    field generation of send_notify, probably caused by a bug in
#    parse_config that leaves everything undefined (perhaps when a
#    new log is added to the config file, perhaps when it's the first
#    log after a begin-host directive?).
# Modified 25 December 2011 by Jim Lippard to use /etc/reportnew.conf as
#    default config file and put default size file in same dir
#    or get it from the config file. Fixed bug mentioned in previous
#    (used "return" instead of "last" to prematurely exit from
#    parse_config subroutine).
# Modified 30 March 2013, 7-8 June 2013 by Jim Lippard to:
#    - Support multiple match/exclude/action triplets per log.
#    - Replace notify: with action: notify
#    - Add action: alert
# Modified 8 June 2013 by Jim Lippard to
#    - Add action: text
# Modified 4 July 2013 by Jim Lippard to add special casing for process
#    accounting logs.
# Modified 18 September 2013 by Jim Lippard to
#    - Add action: execute (with dropped privileges) [incomplete]
# Modified 25 October 2013 by Jim Lippard to change if (defined (@array))
#    to if (exists (@array)), since perl has deprecated the former and now warns.
# Modified 27 November 2019 by Jim Lippard to split hostname and domain name
#    differently (domain name is no longer hardcoded and hostname is just
#    first component of domain name rather than hostname with hardcoded domain
#    name removed).
# Modified 17 February 2020 by Jim Lippard to make notification email sender
#    configurable.
# Modified 22 February 2020 by Jim Lippard to look at rotated logs if a
#    log rotation has occurred since our last check of a standard logfile.
# Modified 23 February 2020 by Jim Lippard to give match_line the option
#    to return the first capture group instead of 1 for a match.
# Modified 24 February 2020 by Jim Lippard to fix bug in checking for first
#    line date stamps which was causing repeat log reports.
# Modified 25 February 2020 by Jim Lippard to add session matching functionality.
# Modified 26-27 February 2020 by Jim Lippard to fix bugs in checking
#    date/time in first line of log (timezone, web log format).
# Modified 27 February 2020 by Jim Lippard to use SHA256 digest of first line
#    of log instead of parsing dates.
# Modified 28 February 2020 by Jim Lippard to assume a single digit number
#    of rotated logs in identify_rotated_logs.
# Modified 2 September 2023 by Jim Lippard to read and write size file
#    before and after each log processed, to set email_sender properly
#    for errors that occur before it is defined, and to track current
#    processing on a log in the size file so that another process doesn't
#    start processing on the same log. Use File::Copy instead of system cp.
# Modified 3 September 2023 by Jim Lippard to fix minor bugs and properly
#    identify gzipped rotated log files.
# Modified 4 September 2023 by Jim Lippard to add new logfiles to size file
#    again, since I broke that.
# Modified 11 November 2023 by Jim Lippard to add macro substitution, both
#    pre-processing (to simplify match/exclude rules) and post-processing
#    (to enrich output by appending macro name or substituting macro name
#    in results). Added global and per-host macros in a single namespace.
# Modified 12 November 2023 by Jim Lippard to add special handling of
#    post-processing macro substitution for IP addresses to avoid appending
#    or substitution in the middle of a larger matching IP address.
# Modified 2 December 2023 by Jim Lippard to use unveil on OpenBSD to restrict
#    file system access to read-only for logs, execute-only for commands, and
#    read/write/create for /tmp.
# Modified 30 December 2023 by Jim Lippard to call pledge correctly.
# Modified 1 January 2024 by Jim Lippard so that first look at process
#    accounting logs in a new year doesn't go to the beginning (call to
#    parsedate was not using PREFER_PAST).
# Modified 1 January 2024 by Jim Lippard to clean up size file code and
#    implement file locking.
# Modified 10 February 2024 by Jim Lippard to allow importing macro values
#    from files.
# Modified 16 March 2024 by Jim Lippard to ignore comment lines in imported
#    files.
# Modified 31 March 2024 by Jim Lippard to properly complain about garbled
#    lines in size file.
# Modified 8 April 2024 by Jim Lippard to fix uninitialized $processing_pid
#    bug when size file is first created, make alert action use line breaks.
# Modified 14 April 2024 by Jim Lippard to handle process accounting logs
#    directly on OpenBSD for greater efficiency.
# Modified 17 April 2024 by Jim Lippard to set last log checktime more
#    appropriately for process accounting logs (to match end of most recent
#    log processing for OpenBSD, and start of most recent log procesing for
#    the old lastcomm/reverse order processing).
# Modified 22 April 2024 by Jim Lippard to remove 'ps' pledge by using
#    kill to verify existence of running processes instead of calling ps.
# Modified 23 April 2024 by Jim Lippard to use IO::Uncompress::Gunzip
#    instead of calling gunzip command, Sys::Hostname instead of hostname
#    command. Use OpenBSD::MkTemp on OpenBSD. Still need exec for sendmail,
#    tai64nlocal/echo (could use Time::TAI64 and Mail::Send; latter requires
#    escaping leading dots on message body lines).
# Modified 5 July 2024 by Jim Lippard to allow signedfile include macros
#    and add signify_publickey global directive.
# Modified 28 July 2024 by Jim Lippard to use Signify.pm.
# Modified 16 December 2024 by Jim Lippard for new config file format and
#    to add -c config check option and -d debug option (for existing
#    debug_mode). Remains backward compatible with the begin-host/end-host
#    format.
# Modified 17 December 2024 by Jim Lippard to fix up -c config check and
#    enforce restriction on using hosts and begin-host/end-host in the same
#    config file.
# Modified 22 December 2024 by Jim Lippard to properly handle process
#    accounting logs on a system that hasn't generated all of the rotated
#    logs yet and fail more gracefully if log files are missing.
# Modified 6 May 2025 by Jim Lippard to add include-macro-file and
#    include-macro-signedfile to allow files of full macro definitions
#    in addition to files of macro values. Create size file if it doesn't
#    exist to reduce unveil surface and pledges.
# Modified 8 May 2025 by Jim Lippard to do additional unveiling for signify
#    checking and use full signify public key path.
# Modified 6 July 2025 by Jim Lippard to use uid in process accounting logs
#    if there's no corresponding user in passwd file.
# Modified 13 August 2025 by Jim Lippard to allow whitespace (not just
#    non-whitespace) in macro definitions.
# Modified 14 September 2025 by Jim Lippard to change path of process
#    accounting log and handle lastcomm format for Linux, add -V version.
# Modified 16 September 2025 by Jim Lippard to support pulling logs from
#    Linux journals using journalctl.
# Modified 18 September 2025 by Jim Lippard to adjust $linux_format for
#    pacct for when there are overlength times.
# Modified 21 September 2025 by Jim Lippard to improve error message that
#    comes from OpenBSD process accounting when it gets turned off, clean
#    up some regexes.
# Modified 4 November 2025 by Jim Lippard to use File::Temp instead of
#    calling mktemp, do better validation on TAI64 dates, avoid shell
#    when invoking sendmail, use chomp instead of chop, create temp dir
#    once per execution, set umask so all created files are only readable
#    by root user, remove tai64nlocal command method for TAI64 log files.
#    Often used ChatGPT for identifying possible vulnerability patterns
#    and for revision assistance from here on, helpful at suggesting
#    patterns but often flawed in implementation details (and terrible
#    at bug finding).
# Modified 12 November 2025 by Jim Lippard to use $macos_format instead of
#    split for macOS process accounting logs, since command names can have
#    spaces in them.
# Modified 15 November 2025 by Jim Lippard to rearrange unveil order slightly,
#    fail if can't create temp dir, improve macOS process accounting sample in
#    config file.
# Modified 22 November 2025 by Jim Lippard to change read_size to get_size
#    to avoid possible implication that it's related to reading from the size
#    file. Do minimal validation on email addresses.
#    Fix minor bug in non-OpenBSD process accounting.
# Modified 25 November 2025 by Jim Lippard to open $JOURNALCTL without
#    invoking shell. Fix bug in identify_rotated_logs not matching gzips.
#    Preparation for privilege separation/running all log checks as
#    _reportnew when run as root.
# Modified 26-29 November 2025 by Jim Lippard to implement privilege
#    separation when run as root and _reportnew user and group exist.
#    Also currently requires use of -p option as well. Privilege
#    separation will parse config and (on OpenBSD) do pledge and unveil,
#    then set up a root listener and fork a child that drops privileges
#    and runs as _reportnew:_reportnew, sending requests to the parent
#    when root access is required to open files, get first line file hashes,
#    or perform Linux journalctl commands. ChatGPT helped refactor here.
# Modified 30 November 2025 by Jim Lippard to add privsep directive to
#    reportnew.conf, fix bugs in privsep, add _reportnew user to process
#    accounting sample configs.
# Modified 1 December 2025 by Jim Lippard to support Linux process
#    accounting's gzipped rotated logs (and better support other OSs).
# Modified 2 December 2025 by Jim Lippard to add some minor security
#    improvements:
#    die if child dropping privs fails, limit priv request length,
#    explicitly block directory traversal attempts, etc.
# Modified 20-21 December 2025 by Jim Lippard to build device name cache
#    for OpenBSD and macOS and make direct process accounting file parsing
#    work on Linux and macOS. Linux was easy, macOS less so.
# Modified 21 December 2025 by Jim Lippard to fix bugs in Linux handling of
#    checktime in process accounting. ChatGPT was not helpful.
# Modified 22 December 2025 by Jim Lippard to read larger block of records
#    in process accounting (all platforms) to improve efficiency.
# Modified 24 December 2025 by Jim Lippard to use SHA256 hash of first
#    record to verify that the same file is being processed across runs.
# Modified 31 December 2025 by Jim Lippard to precompile match/exclude
#    regexes for efficiency. ChatGPT provided assistance.
# Modified 1 January 2026 by Jim Lippard to fix an edge case in session
#    handling with rotated logs. ChatGPT couldn't find this.
# Modified 2 January 2026 by Jim Lippard to formalize interprocess comms
#    using JSON and HMACs. Add require_module. Used Claude to assist
#    with this refactoring. The HMACs are really unnecessary for any feasible
#    threat model given what this script does, but was fun.
# Modified 3 January 2026 by Jim Lippard to make replay attack mitigation
#    stricter. Used variant of ChatGPT recommendation. Again this is likely
#    massive overkill for this script.
# Modified 4 January 2026 by Jim Lippard to not require Signify unless
#    signed files are used. Remove & on subroutine calls.
# Modified 4 January 2026 by Jim Lippard to not export anything from
#    Signify, only allow one privsep: in the config file, and strip
#    carriage returns from mailed log output if found.
# Modified 8 February 2026 by Jim Lippard (with Claude) to complete
#    implementation of 'execute' action. Scripts must be in a scripts
#    subdirectory under the reportnew directory and must be signed with
#    the signify key defined in the config, and are executed in the
#    unprivileged child process (_reportnew:_reportnew) if privsep is
#    used and as nobody:nogroup otherwise. If reportnew is already
#    being run from a nonprivileged account, scripts are run from that
#    same account. Since scripts must be signed to execute, I didn't
#    bother creating a separate privileged script execution handler
#    for privsep to run scripts as a different user. The main risk in
#    this case is that a script could clobber the size file, which
#    _reportnew:_reportnew has write access to. (Another alternative
#    would be to make that root-owned and make all use of the size
#    file run through the privileged process, which would be relatively
#    straightforward.)
# Modified 11 February 2026 by Jim Lippard (using Claude) to add an
#    optional "execute" action to a notify/text/alert action.
# Modified 19 February 2026 by Jim Lippard to wrap log lines > 990
#    characters in length (RFC 5321 has a 998-char limit).
# Modified 25 February 2026 by Jim Lippard for minor change to Linux
#    process flags.
# Modified 27 February 2026 by Jim Lippard to remove tmppath pledge.
# Modified 4 March 2026 by Jim Lippard to use Z24 instead of A24 for
#    unpacking command names in process accounting logs, to avoid
#    unnecessary (and sendmail-confusing) nulls.
# Modified 16 March 2026 by Jim Lippard to fix bug in process accounting
#    processing on Linux.
# Modified 20 March 2026 by Jim Lippard with Claude assistance to
#    implement time range checking and explicitly require exclude
#    directives in every match/exclude/action triplet.
# Modified 22 March 2026 by Jim Lippard to provide better validation
#    and error reporting on invalid time specifications and to fix bug
#    in keeping time constraints aligned with match/exclude/action
#    triplets when many rules have no "times:" directives.
# Modified 23 March 2026 by Jim Lippard to a a shorthand for time
#    constraints range every hour or half hour.
# Modified 13 April 2026 by Claude at the direction of Jim Lippard to
#    add "all except <host list>" for the "hosts:" directive. Fix -c
#    config check to not complain about the nonexistence, nonreadability,
#    nonexecutability, and non-signedness or bad signatures of files for
#    other hosts.
# Modified 16 April 2026 by Jim Lippard to move umask earlier so size
#    file is mode 0600.
# Modified 2 May 2026 by Jim Lippard to fix just over a dozen minor issues
#    identified by Claude security and error assessment (Opus 4.7).
# Modified 3 May 2026 by Jim Lippard to fix long-standing double dereference
#    issue that was a latent bug for configs with rules without action:
#    directives (that use default). Use block eq greps for Linux journal
#    unit and syslog facility checks.
# Modified 12 May 2026 by Jim Lippard to rename processes when using
#    privilege separation.
# Modified 15 May 2026 by Jim Lippard to shorten process names.
# Modified 16 May 2026 by Jim Lippard to pass epoch time to journalctl --since.
#    (Bug identified by Gemini, along with five false positives; was passing
#    malformed time string with erroneous fractional seconds which were likely
#    being ignored.)

# Suggested enhancements:
# * Process all rotated logs and original log, as well as all components
#   of cyclogs or multilogs, together and process all notifications for them
#   together at once instead of once per match per file.
# * Allow customization of subject line so that multiple reportnew
#   configs can be used on the same machine/same logs and be distinguishable.
# * Allow to run continuously (like swatch) monitoring logs with
#   select.  That will perhaps be more efficient than starting up
#   a perl process every N minutes, and will catch log changes more
#   rapidly.  It should wait a little bit for additional matching
#   messages, though, so that it doesn't send a separate message for
#   each log line.  (Easiest way might be to make it go into an
#   infinite loop, sleeping every N minutes at the end.  Though
#   it would be more efficient to use select.)

### Required packages.

use strict;
use warnings;
use feature 'state';
use Digest::SHA qw( hmac_sha256_hex sha256_hex );
use Fcntl qw(:DEFAULT :flock);
use File::Basename;
use File::Copy;
use File::Find;
use if $^O ne 'openbsd', 'File::Temp', qw( :mktemp tempfile );
use Getopt::Std;
use IO::Uncompress::Gunzip qw( gunzip $GunzipError);
use POSIX qw( ctime fmod sysconf _SC_CLK_TCK );
use Sys::Hostname;
use Time::HiRes qw( gettimeofday );
use Time::ParseDate;
# Required for privilege separation:
# IO::FDPass
# Privileges::Drop
# JSON::MaybeXS (or JSON::PP as a fallback, slower but standard)
use base; # required for Privileges::Drop
use English; # required for Privileges::Drop
use IO::Socket::UNIX; # required for socketpair call
# required for multilog format
#use Time::TAI64 qw( :tai64n );
#my $TimeTAI64_module = 1;
my $TimeTAI64_module = 0;

use if $^O eq 'openbsd', 'OpenBSD::MkTemp', qw( mkdtemp mkstemp );
use if $^O eq 'openbsd', 'OpenBSD::Unveil';
use if $^O eq 'openbsd', 'OpenBSD::Pledge';

### Constants.

# for process accounting
my $AHZ = accounting_hz();
use constant SECSPERHOUR => 60 * 60;
use constant SECSPERMIN => 60;

# for privsep/interprocess
use constant {
    RESP_SUCCESS => 'success',
    RESP_ERROR => 'error',
    RESP_FD_FOLLOWS => 'fd_follows' # file descriptor will be sent next
};

my $HOSTNAME = hostname();
my ($SHORT_HOSTNAME, $DOMAINNAME) = split (/\./, $HOSTNAME, 2);

### Set to your security admin.
my $SECURITY_ADMIN = 'lippard@discord.org';

my $EMAIL_SENDER = 'nobody@' . $DOMAINNAME;

# for Linux journal logs.
my $JOURNALCTL = '/usr/bin/journalctl';
my $SYSTEMCTL = '/usr/bin/systemctl';

my $LASTCOMM = '/usr/bin/lastcomm';
my $SENDMAIL = '/usr/sbin/sendmail';
my $SIGNIFY = '/usr/bin/signify';
my $ETC_DIR = '/etc';
my $ZONEINFO_DIR = '/usr/share/zoneinfo';

my $VERSION = 'reportnew 1.35b of 16 May 2026';

my $DEFAULT_CONFIG_DIR = '/etc/reportnew';
my $DEFAULT_CONFIG_NAME = 'reportnew.conf';
my $DEFAULT_SIZE_FILE_DIR = '/etc/reportnew';
my $DEFAULT_SIZE_FILE_NAME = 'reportnew.size';
my $CONFIG_SUFFIX = ".conf";
my $SIZE_SUFFIX = ".size";

my ($MACRO_DIR, $SCRIPT_DIR); # config_dir and config_dir/scripts

my $SIZE_FILE_LOCK_TIMEOUT_LIMIT = 10;

my $SIGNIFY_DIR = '/etc/signify';

my $PROCESS_ACCOUNTING_LOG = '/var/account/acct'; # BSD and macOS location.
$PROCESS_ACCOUNTING_LOG = '/var/log/account/pacct' if ($^O eq 'linux');
my $MAX_PROCESS_ACCOUNTING_FILE = 3;
$MAX_PROCESS_ACCOUNTING_FILE = 9 if ($^O eq 'linux'); # not really a max

# set to 1 if you need to use lastcomm instead of processing accounting
# files directly.
my $USE_LASTCOMM = 0;
$USE_LASTCOMM = 1 if ($^O ne 'openbsd' && $^O ne 'linux' && $^O ne 'macos');

my $NL = '
';

my $LOG_TYPE_STANDARD_LOG = 0;
my $LOG_TYPE_CYCLOG_OR_MULTILOG = 1;
my $LOG_TYPE_PROCESS_ACCOUNTING = 2;
my $LOG_TYPE_LINUX_JOURNAL = 3;

my $LOG_PROCESSING_START = 1;
my $LOG_PROCESSING_END = 2;
my $LOG_APPEND = 3;

my $GLOBAL_CONTEXT = 1;
my $HOST_CONTEXT = 2;
my $SKIPPING = 3;
my $LOG_CONTEXT = 3; # not used, uses !defined($current_logfile)

my @INITIAL_PROMISES = ('unveil');
my @NONPRIVSEP_PROMISES = ('rpath', 'wpath', 'cpath', 'flock', 'exec', 'proc');
my @PRIVSEP_INITIAL_CHILD_PROMISES = ('id', 'prot_exec');
my @PRIVSEP_CHILD_PROMISES = ('recvfd');
my @PRIVSEP_PARENT_PROMISES = ('chown', 'sendfd');

### Variables.

# Filenames.
use vars qw($config_file $size_file $temp_dir);

# Global variables from config file.
use vars qw(
    $master_notify
    $email_sender
    $signify_pubkey
    %time_ranges
    @logfiles
    %match_hash
    %exclude_hash
    %action_hash
    %match_hash_ref
    %exclude_hash_ref
    %action_hash_ref
    %time_hash
    %execute_scripts
    %global_preproc_macro
    $have_global_postproc_macros
    %global_append_macro
    %global_substitute_macro   
    %preproc_macro
    $have_postproc_macros
    %append_macro
    %substitute_macro
    );

# Global variables from size file.
use vars qw(
    %log_size
    %log_mtime
    %log_checktime
    %log_sha256_digest
    %log_processing_pid
    );

# Global variables for linux journal logs.
use vars qw(
    @linux_journal_units
    @linux_journal_syslog_facilities
);

# Other global variables.
use vars qw(
    $debug_mode
    $config_check
    %opts
    %defined_hosts
    %defined_hostlog
    %devname_cache
    );

# Local variables in main program.
my ($logfile, $size, $mtime, $sha256_digest,  @cyclog_files, $cyclog_file,
    $old_log_checktime, $got_first_cyclog_file, $temp_logfile);
my ($rotated_logs_flag, $processed_rotated_log_flag, @rotated_logs, $rotated_logfile);

# For privilege separation.
my ($user, $reporter_uid, $reporter_gid, $use_privsep,
    $privsep_flag, $priv_flag, $nonpriv_flag,
    $parent_sock, $child_sock, $HMAC_SECRET);

### Main program.

getopts ('cdpV', \%opts) || die "Usage: reportnew [-c (check)|-d (debug)|-V (version)] [config-file]\n";

$config_check = $opts{'c'};
$debug_mode = $opts{'d'} || $config_check; # config_check implies debug_mode;
$use_privsep = $opts{'p'}; # can override default or "no" in config.
if ($opts{'V'}) {
    die "-V (version) cannot be used with other options.\n" if ($config_check || $debug_mode);
    print "$VERSION\n";
    exit;
}

if ($#ARGV == 0) {
    $config_file = $ARGV[0];
}
elsif ($#ARGV < 0) {
    $config_file = "$DEFAULT_CONFIG_DIR/$DEFAULT_CONFIG_NAME";
}
else {
    die "Usage: reportnew [-c (check)|-d (debug)|-V (version)] [config-file]\n";
}

$MACRO_DIR = dirname ($config_file);
$SCRIPT_DIR = dirname ($config_file) . '/scripts';

if (substr ($config_file, length ($config_file) - 5, 5) ne $CONFIG_SUFFIX) {
    $config_file .= $CONFIG_SUFFIX;
}

# Sanitize environment
BEGIN {
    $ENV{PATH} = '/usr/bin:/bin';
    delete @ENV{qw(IFS CDPATH ENV BASH_ENV PERL5LIB LD_PRELOAD LD_LIBRARY_PATH)};
}

# See if Signify is available.
my $have_signify = 0;
$have_signify = 1 if (require_module ('Signify'));

# If on OpenBSD, use pledge and unveil. Do as much as possible before
# parsing the config, then finish unveiling after the config is parsed.
# We also build %devname_cache for OpenBSD before unveiling.
if ($^O eq 'openbsd') {
    # Pledge. stdio is already included.
    # @INITIAL_PROMISES: unveil
    # @NONPRIVSEP_PROMISES: rpath, wpath, cpath, flock, proc, exec
    #     flock for locking size file
    #     proc for pid testing and exec
    #     exec for sendmail
    # @PRIVSEP_INITIAL_CHILD_PROMISES: id, prot_exec - for dropping privs
    # @PRIVSEP_CHILD_PROMISES: recvfd - for receiving file descriptors
    # @PRIVSEP_PARENT_PROMISES: sendfd - for sending file descriptors
    pledge (@INITIAL_PROMISES, @NONPRIVSEP_PROMISES,
	    @PRIVSEP_PARENT_PROMISES,
	    @PRIVSEP_INITIAL_CHILD_PROMISES,
	    @PRIVSEP_CHILD_PROMISES) || die "Cannot pledge promises. $!\n";

    # Build %devname_cache on OpenBSD.
    build_devname_cache() if ($^O eq 'openbsd');
    
    # Needed to parse dates on process accounting logs.
    unveil ($ZONEINFO_DIR, 'r');

    # Need for username resolution on process accounting logs.
    unveil ($ETC_DIR, 'r');

    # Unveil directory where config file is, so that any macros
    # with file import can be processed.
    unveil ($MACRO_DIR, 'r');

    # Allow writing to and creating files and dirs in /tmp.
    unveil ('/tmp', 'rwc');

    # Allow execution of commands. $LASTCOMM excluded since
    # it's not used on OpenBSD.
    unveil ($SENDMAIL, 'x');
    # Signify for signed include files in macros.
    unveil ($SIGNIFY, 'rx');
    unveil (dirname ($SIGNIFY), 'rx');
    unveil ($SIGNIFY_DIR, 'rx');
    unveil ('/dev/null', 'rw');
}

parse_config ($config_file);

if ($config_check) {
    print "No issues identified in config file.\n";
    exit;
}

# Build %devname_cache on macOS.
build_devname_cache() if ($^O eq 'darwin');

# Determine if we're going to use privilege separation.
# Conditions: If running as root and _reportnew user and group exist.
$user = $ENV{'LOGNAME'} || $ENV{'USER'} || $ENV{'USERNAME'} || 'unknown';
$reporter_uid = getpwnam ('_reportnew');
$reporter_gid = getgrnam ('_reportnew');

# Will quietly not use privsep if not run as root or _reportnew account doesn't exist.
if ($use_privsep && $user eq 'root' &&
    defined ($reporter_uid) &&
    defined ($reporter_gid)) {
    $privsep_flag = 1;
    $priv_flag = 0;
    $nonpriv_flag = 0;

    # Needed for privsep HMAC.
    unveil ('/dev/urandom', 'rw') if ($^O eq 'openbsd');

    # Unveil /usr/local/libdata/perl5 if on OpenBSD to allow
    # loading these modules at runtime.
    unveil ('/usr/local/libdata/perl5', 'r') if ($^O eq 'openbsd');
    unveil ('/usr/libdata/perl5', 'r') if ($^O eq 'openbsd'); # for JSON::PP

    # Load required modules.
    require_module ('IO::FDPass')
	or die "Could not require IO::FDPass. $@\n";
    require_module ('Privileges::Drop')
	or die "Could not require Privileges::Drop. $@\n";

    # Try to load JSON (with fallback).
    my $json_loaded = 0;
    if (require_module ('JSON::MaybeXS', qw(encode_json decode_json))) {
	$json_loaded = 1;
	print "DEBUG: Using JSON::MaybeXS\n" if ($debug_mode);
    }
    elsif (require_module ('JSON::PP', qw(encode_json decode_json))) {
	$json_loaded = 1;
	print "DEBUG: Using JSON::PP\n" if ($debug_mode);
    }

    die "No JSON module available for privilege separation.\n" unless $json_loaded;
}
elsif ($^O eq 'openbsd') { # not using privsep
    # If we have execute scripts, we need 'id' and 'prot_exec' promises to
    # drop privileges when executing them (since we're not using privsep).
    if (keys %execute_scripts) {
	pledge (@INITIAL_PROMISES, @NONPRIVSEP_PROMISES, @PRIVSEP_INITIAL_CHILD_PROMISES) || die "Cannot pledge promises, non-privsep with script execution. $!\n";
    }
    else {
        pledge (@INITIAL_PROMISES, @NONPRIVSEP_PROMISES) || die "Cannot pledge promises, non-privsep. $!\n";
    }
}

# Unveil log dirs from config and lock.
# Also unveil execute scripts directory if any execute actions are defined.
if ($^O eq 'openbsd') {
    my (%unveiled_dirs, $logdir, $size_file_dir);

    # Unveil log directories.
    foreach $logfile (@logfiles) {
	next if $logfile =~ /^journal /; # shouldn't happen.
	$logdir = dirname ($logfile);
	if (!defined ($unveiled_dirs{$logdir})) {
	    unveil ($logdir, "r");
	    $unveiled_dirs{$logdir} = 1;
	}
    }

    # Unveil execute scripts directory if any execute actions are defined.
    # Use "rwx" to allow scripts to read, write and execute within the
    # directory, no "c" (create) - admin must create any necessary data
    # subdirectories.
    if (keys %execute_scripts) {
	unveil ($SCRIPT_DIR, "rwx");
    }

    # Need to unveil containing dir of size file, which can but
    # need not be same as containing dir of config file.
    $size_file_dir = dirname ($size_file);
    unveil ($size_file_dir, 'r');
    
    # Allow reading from and writing to size file.
    # c required for append.
    unveil ($size_file, 'rwc');

    # Don't lock yet.
}

# Any files we create are rw for root (or running user, non-priv works
# fine for process accounting on non-Linux) only.
umask 0077;

# Need to temporarily unveil containing dir with rwc in order to create
# size file if it doesn't already exist. Go ahead and create it.
if (!-e $size_file) {
    my $size_file_dir = dirname ($size_file);
    unveil ($size_file_dir, 'rwc') if ($^O eq 'openbsd');
    if (open (SIZEFILE, '>', $size_file)) {
	close (SIZEFILE);
    }
    else {
	send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot create size file. $!");
    }
    # If using privilege separation, give _reportnew:_reportnew ownership.
    chown ($reporter_uid, $reporter_gid, $size_file) if ($privsep_flag);

    # Remove unneeded access from dir.
    unveil ($size_file_dir, 'r') if ($^O eq 'openbsd');
}

# Lock down unveiling. Equivalent to removing pledge for @INITIAL_PROMISES
# (which so far only contains 'unveil').
unveil() if ($^O eq 'openbsd');
    
# Create a temp_dir.
$temp_dir = mkdtemp ('/tmp/reportnew.XXXXXXX') || die "Cannot create temp dir. $!\n";
chomp ($temp_dir);

# Privilege separation, if running as root:
# Parent will set up listener to access files requiring root access,
# child will drop privileges and run as _reportnew:_reportnew.
if ($privsep_flag) {
    
    # Create socket for interprocess communications.
    ($parent_sock, $child_sock) = IO::Socket::UNIX->socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC)
	or die "socketpair: $!";

    # Enable autoflush on both sockets to prevent buffering issues.
    my $old_fh = select ($parent_sock); $| = 1; select ($old_fh);
    $old_fh = select ($child_sock); $| = 1; select ($old_fh);

    # Create shared secret.
    $HMAC_SECRET = generate_random_secret();

    # Fork.
    my $pid = fork();
    die "fork: $!" unless defined $pid;

    # Parent, privileged listener.
    if ($pid != 0) {
	close $child_sock;

	# Rename process.
	$0 = 'reportnew [priv]';

	print "DEBUG: child proc=$pid, parent proc=$$\n" if ($debug_mode);
	# Don't need @PRIVSEP_CHILD_PROMISES or @PRIVSEP_INITIAL_CHILD_PROMISES.
	if ($^O eq 'openbsd') {
	    pledge (@NONPRIVSEP_PROMISES, @PRIVSEP_PARENT_PROMISES) || die "Cannot pledge promises. $!\n"
	}

	# This is the privileged process.
	$priv_flag = 1;

	# chown temp_dir to _reportnew so child has access.
	chown ($reporter_uid, $reporter_gid, $temp_dir);

	# Listen for requests that require privileges.
	priv_listener();

	# Shutdown after EOF.
	exit;
    }

    # Child.
    close $parent_sock;

    # Rename process.
    $0 = 'reportnew log processor';
    
    # This call will die if it fails.
    print "DEBUG: Privileges::Drop will always fail on macOS with perl 5.34.1 unless patched.\n" if ($debug_mode && $^O eq 'darwin' && $^V eq 'v5.34.1');
    Privileges::Drop::drop_uidgid ($reporter_uid, $reporter_gid);
    print "DEBUG: real uid: $<, eff uid: $>, real gid: $(, eff gid: $)\n" if ($debug_mode);

    $nonpriv_flag = 1;

    # No longer need @PRIVSEP_INITIAL_CHILD_PROMISES.
    if ($^O eq 'openbsd') {
	    pledge (@NONPRIVSEP_PROMISES, @PRIVSEP_CHILD_PROMISES) || die "Cannot pledge promises, nonpriv child. $!\n";
    }
}

# Modified to read and write size file after each log processed, so that
# if something fails on processing, work already done isn't repeated.

foreach $logfile (@logfiles) {

    # Read all logs from size file, skip this one if it's already
    # being processed, otherwise mark this one as being processed.
    read_size_file ($size_file);
    next if ($log_processing_pid{$logfile} != 0 && pid_exists ($log_processing_pid{$logfile}));
    write_size_file ($size_file, $logfile, $LOG_PROCESSING_START); # save PID

    if ($logfile eq $PROCESS_ACCOUNTING_LOG) {
	# For process accounting logs, we get the log checktime from this
	# read and not at the end of processing the log.
	
	# Still create a temp logfile from process accounting logs,
	# but process them directly instead of by calling lastcomm.
	# Formerly OpenBSD only, now also works for Linux and macOS.
	if (!$USE_LASTCOMM) {
	    ($temp_logfile, $log_size{$logfile}, $log_mtime{$logfile},
	     $log_checktime{$logfile}, $log_sha256_digest{$logfile}) =
		direct_read_process_acct_log ($PROCESS_ACCOUNTING_LOG,
					       $log_size{$logfile},
					       $log_checktime{$logfile},
					       $log_sha256_digest{$logfile},
					       $temp_dir);
	}
	# Otherwise:
	# Read process accounting logs out to the last time seen, or
	# for all of it, and write it out to a tmp file, $temp_logfile,
	# putting it into chronological order instead of reverse.
	# Runs on the privileged side with privsep on Linux.
	else {
	    ($temp_logfile, $log_checktime{$logfile}) = read_process_acct_log ($PROCESS_ACCOUNTING_LOG, $log_checktime{$logfile}, $temp_dir);
	}
	# Then use check_logfile on the temp file, disregarding the size,
	# which is set to 0, just as for files in cyclogs.
	my $ignore_checktime; # using the checktime from the read, not the check
	my $ignore_mtime; # using the mtime from the read, not the check.
	my $ignore_size; # using the log size from the read, not the check
	($ignore_size, $ignore_mtime, $ignore_checktime, $log_sha256_digest{$logfile}) =
	    check_logfile ($temp_logfile, 0, $log_mtime{$logfile},
			    $log_checktime{$logfile},
			    $log_sha256_digest{$logfile},
			    $match_hash_ref{$logfile},
			    $exclude_hash_ref{$logfile},
			    $action_hash_ref{$logfile},
			    $LOG_TYPE_PROCESS_ACCOUNTING);

	# Unlink $temp_logfile.
	unlink ($temp_logfile);
    }
    elsif ($logfile =~ /^journal /) {
	# Similar to process accounting files, we handle Linux journal logs
	# by writing out the relevant information to a temporary log file,
	# and don't care about tracking size. We use the last read time as
	# the start time (with journalctl --since).
	# Options:
	# journal unit <unit>
	#    journalctl -u <unit> --since <last_check_time>
	# journal syslog-id <identifier>
	#    journalctl -t <identifier> --since <last_check_time>
	# journal syslog-facility <facility>
	#    journalctl --facility=<facility> --since <last_check_time>
	($temp_logfile, $log_checktime{$logfile}) = read_linux_journal ($logfile, $log_checktime{$logfile}, $temp_dir);
	# Then use check_logfile on the temp file, disregarding the size,
	# which is set to 0, just as for files in cyclogs.
	my $ignore_checktime;
	($log_size{$logfile}, $log_mtime{$logfile}, $ignore_checktime, $log_sha256_digest{$logfile}) =
	    check_logfile ($temp_logfile, 0, $log_mtime{$logfile},
			    $log_checktime{$logfile}, $log_sha256_digest{$logfile},
			    $match_hash_ref{$logfile},
			    $exclude_hash_ref{$logfile}, $action_hash_ref{$logfile}, $LOG_TYPE_LINUX_JOURNAL, $logfile);

	# Unlink $temp_logfile.
	unlink ($temp_logfile);
    }
    elsif (!-d $logfile) { # Standard syslog file.
	# If there are archived rotated logs which have been modified since
	# log_checktime{$logfile}, we should check any contents that postdate
	# that time. The oldest rotated logfile changed after our last
	# checktime will be the last one we looked at OR a more recent one
	# that we have to look at in its entirety (depending on frequency
	# of checks vs log rotation/retention, it might have already been
	# deleted).
	# We check the oldest one modified after our last check, and
	# if its first line is newer than our last check we check the entire
	# thing, otherwise we presume it's the same logfile we last checked
	# and we start where we left off.
	($rotated_logs_flag, @rotated_logs) = identify_rotated_logs ($logfile, $log_checktime{$logfile});
	# We have rotated logs we need to examine.
	if ($rotated_logs_flag) {
	    $processed_rotated_log_flag = 0;
	    foreach $rotated_logfile (@rotated_logs) {
		# If the oldest logfile is one we've seen part of before,
		# we'll seek to the right position. We verify by checking
		# the SHA256 digest of the first line to see if it matches
		# what was there before. This check also occurs in check_logfile.
		# This will "work" on a gzip but could have high rate of
		# false positives.
		if (!$processed_rotated_log_flag) {
		    # If first line is different from what we last saw,
		    # then start from the beginning.
		    if ($log_sha256_digest{$logfile} eq '' ||
			first_log_line_sha256_digest ($rotated_logfile) ne
			$log_sha256_digest{$logfile}) {
			$log_size{$logfile} = 0;
		    }
		    # We don't look at return values here.
			check_logfile ($rotated_logfile, $log_size{$logfile}, $log_mtime{$logfile},
					$log_checktime{$logfile},
					$log_sha256_digest{$logfile},
					$match_hash_ref{$logfile},
					$exclude_hash_ref{$logfile},
					$action_hash_ref{$logfile},
					$LOG_TYPE_STANDARD_LOG);
		    
			$processed_rotated_log_flag = 1;
		}
		else {
		    # Again, not looking at return values.
		    check_logfile ($rotated_logfile, 0, 0, 0,
				    $log_sha256_digest{$logfile},
				    $match_hash_ref{$logfile},
				    $exclude_hash_ref{$logfile},
				    $action_hash_ref{$logfile},
				    $LOG_TYPE_STANDARD_LOG);
		}

		# Set log_size to 0 for the regular logfile since we need to
		# look at the whole thing, and update first line SHA256.
		$log_size{$logfile} = 0;
		$log_sha256_digest{$logfile} = first_log_line_sha256_digest ($logfile);
	    } # End processing of rotated logs.
	} # Standard logfile check.
	
	($log_size{$logfile}, $log_mtime{$logfile},
	 $log_checktime{$logfile}, $log_sha256_digest{$logfile}) =
	    check_logfile ($logfile, $log_size{$logfile},
			    $log_mtime{$logfile},
			    $log_checktime{$logfile},
			    $log_sha256_digest{$logfile},
			    $match_hash_ref{$logfile},
			    $exclude_hash_ref{$logfile},
			    $action_hash_ref{$logfile},
			    $LOG_TYPE_STANDARD_LOG);
    }
    else { # cyclog or multilog
	@cyclog_files = get_cyclog_files ($logfile);
	if ($!) {
	    send_error ($logfile, "Could not open cyclog/multilog directory for $logfile. $!");
	}
	elsif (!@cyclog_files) {
	    send_error ($logfile, "Empty cyclog/multilog $logfile.");
	}
	$got_first_cyclog_file = 0;
	$old_log_checktime = $log_checktime{$logfile};
	foreach $cyclog_file (sort (@cyclog_files)) {
	    next if ($cyclog_file eq 'lock'); # multilog format
	    next if ($cyclog_file eq 'state'); # multilog format
	    $cyclog_file = $logfile . '/' . $cyclog_file;
	    ($size, $mtime) = get_size ($cyclog_file);
	    next if ($mtime < $old_log_checktime);
	    # If we get here, then we've found the oldest changed file
	    # (since we last checked).
	    $got_first_cyclog_file++;
	    if ($got_first_cyclog_file == 1) {
		($log_size{$logfile}, $log_mtime{$logfile},
		 $log_checktime{$logfile}, $log_sha256_digest{$logfile}) =
		    check_logfile ($cyclog_file, $log_size{$logfile},
				    $log_mtime{$logfile},
				    $old_log_checktime,
				    $log_sha256_digest{$logfile},
				    $match_hash_ref{$logfile},
				    $exclude_hash_ref{$logfile},
				    $action_hash_ref{$logfile},
				    $LOG_TYPE_CYCLOG_OR_MULTILOG);
	    }
	    # For all the new files, we don't care about size or mtime.
	    else {
		($log_size{$logfile}, $log_mtime{$logfile},
		 $log_checktime{$logfile}) =
		    check_logfile ($cyclog_file, 0, 0, 0,
				    $log_sha256_digest{$logfile},
				    $match_hash_ref{$logfile},
				    $exclude_hash_ref{$logfile},
				    $action_hash_ref{$logfile},
				    $LOG_TYPE_CYCLOG_OR_MULTILOG);
		# If we're on the last one, set the first line SHA256 digest.
		# (Count is number of last element, not number of elements.)
		# This is never reached if there is only one, but we've
		# already set SHA256 above for the first one (zeroth) as well.
		if ($got_first_cyclog_file > $#cyclog_files) {
		    $log_sha256_digest{$logfile} = first_log_line_sha256_digest ($cyclog_file);
		}
	    } # foreach
	}
    }

    # Read all other logs from size file, then update this one.
    read_size_file ($size_file, $logfile);
    write_size_file ($size_file, $logfile, $LOG_PROCESSING_END);
}

# Remove temp dir.
rmdir ($temp_dir);

### Subroutines.

# Subroutine to assist in optional module importing.
# (Also used in sigtree.pl.)
sub require_module {
    my ($module, @imports) = @_;

    if (!eval "require $module; 1") {
	return 0;
    }

    if (@imports) {
	$module->import (@imports);
    }

    return 1;
}

## Config parsing and related subroutines to validate config elements.

# Subroutine to parse configuration file.
# As written you the global directives at the top in the sample config file
# can be anywhere, it would probably be better to have a global section
# and individual host sections.
sub parse_config {
    my ($config_file) = @_;
    my ($current_context, $directive, $value, $rule, $rule_type, $rule_text,
	$line_num, $current_logfile, $specified_host, $current_host,
	$all_host_master_notify, $action, $action_value,
	$current_match_session_match_flag);
    my ($macro_name, $macro_value, $macro_options, $macro_file);
    my ($multihost_config, $oldstyle_multihost, $found_my_host);
    my ($linux_journal_unit, $linux_journal_syslogid,
	$linux_journal_syslogfacility);
    my $have_privsep = 0;

    $line_num = 0;
    $specified_host = 0;
    $current_match_session_match_flag = 0;
    $current_context = $GLOBAL_CONTEXT;
    $have_global_postproc_macros = 0;
    $have_postproc_macros = 0;
    $multihost_config = 0;
    $oldstyle_multihost = 0;
    $found_my_host = 0;
    $linux_journal_unit = 0;
    $linux_journal_syslogid = 0;
    $linux_journal_syslogfacility = 0;
    if (open (CONFIG, '<', $config_file)) {
	while (<CONFIG>) {
	    $line_num++;
	    if (!/^\s*#|^\s*$/) {
		chomp;
		# if SKIPPING, don't resume until we see another "hosts:",
		# unless doing a config_check.
		next if ($current_context == $SKIPPING &&
			 !/^\s*hosts:/ &&
			 !$config_check);
		# Macro definitions.
		if (/^([\w\-]+)\s*=\s*\"(.+)\"(.*$)/) {
		    $macro_name = $1;
		    $macro_value = $2;
		    $macro_options = $3;
		    parse_macro ($macro_name, $macro_value, $macro_options,
				  $current_context, $current_host,
				  $line_num, 0);
		}
		elsif (/^.*:.*$/) {
		    ($directive, $value) = split (/:\s*/, $_, 2);

		    if ($directive eq 'hosts') {
			if ($multihost_config && $oldstyle_multihost) {
			    send_error ($config_file, "Found \"hosts:\" directive while using old style \"begin-host:\" and \"end-host:\" directives on line $line_num. $value");
			    exit;
			}
			elsif (!$multihost_config) {
			    $multihost_config = 1;
			}
			if (defined ($master_notify) && !defined ($current_host)) {
			    $all_host_master_notify = $master_notify;
			}
			$current_host = $value if ($config_check);
			if (my_host ($line_num, $value)) {
			    $current_context = $HOST_CONTEXT;
			    $current_host = $HOSTNAME if (!$config_check);
			    $found_my_host = 1;
			}
			else {
			    $current_context = $SKIPPING;
			}
		    }
		    elsif ($directive eq 'begin-host') {
			if ($multihost_config && !$oldstyle_multihost) {
			    send_error ($config_file, "Found \"begin-host:\" directive while using new style \"hosts:\" directive on line $line_num. $value");
			    exit;
			}
			elsif (!$multihost_config) {
			    $multihost_config = 1;
			    $oldstyle_multihost = 1;
			}
			if (defined ($master_notify) && !defined ($current_host)) {
			    $all_host_master_notify = $master_notify;
			}
			$current_host = $value;
			$current_context = $HOST_CONTEXT;
			if ($config_check && !defined ($defined_hosts{$current_host})) {
			    print "New host name in \"begin-host:\" directive on line $line_num. $current_host\n";
			    $defined_hosts{$current_host} = 1;
			}
		    }
		    elsif ($directive eq 'end-host') {
			if ($multihost_config && !$oldstyle_multihost) {
			    send_error ($config_file, "Found \"end-host:\" directive while using new style \"hosts:\" directive on line $line_num. $value");
			    exit;
			}
			if ($current_host ne $value) {
			    send_error ($config_file, "end-host directive does not match begin-host directive (which uses \"$current_host\") on line $line_num. $value\n");
			    exit;
			}
			if ($current_host eq $HOSTNAME || $current_host eq $SHORT_HOSTNAME) {
			    $found_my_host = 1;
			    last unless ($config_check);
			}
			$current_host = "";
			undef $master_notify;
			undef @logfiles;
			undef %match_hash;
			undef %exclude_hash;
			undef %action_hash;
			undef %preproc_macro;
			undef %append_macro;
			undef %substitute_macro;
			$have_postproc_macros = 0;
		    }
		    elsif ($directive eq 'master_notify') {
			if (defined ($master_notify)) {
			    send_error ($config_file, "Second master_notify directive on line $line_num. $value");
			    exit;
			}
			if (!valid_email ($value)) {
			    send_error ($config_file, "Invalid email address in master_notify directive on line $line_num. $value");
			    exit;
			}
			$master_notify = $value;
		    }
		    elsif ($directive eq 'size_file') {
			if (defined ($size_file)) {
			    send_error ($config_file, "Second size_file directive on line $line_num. $value");
			    exit;
			}
			$size_file = $value;
		    }
		    elsif ($directive eq 'email_sender') {
			if (defined ($email_sender)) {
			    send_error ($config_file, "Second email_sender directive on line $line_num. $value");
			    exit;
			}
			$email_sender = $value;
		    }
		    elsif ($directive eq 'privsep') {
			if ($have_privsep) {
			    send_error ($config_file, "Second privsep directive on line $line_num. $value");
			    exit;
			}
			$have_privsep = 1; # $use_privsep could come from -p
			if ($value eq 'yes') {
			    $use_privsep = 1;
			}
			elsif ($value eq 'no') {
			    $use_privsep = 0;
			}
			else {
			    send_error ($config_file, "Invalid value for privsep directive on line $line_num, must be \"yes\" or \"no\". $value");
			    exit;
			}
		    }
		    elsif ($directive eq 'signify_pubkey') {
			if (defined ($signify_pubkey)) {
			    send_error ($config_file, "Second signify_pubkey directive on line $line_num. $value");
			    exit;
			}
			$signify_pubkey = $value;
			if ($signify_pubkey !~  /^[\w\-\.]+$/) {
			    send_error ($config_file, "Invalid signify_pubkey file name \"$signify_pubkey\" line $line_num. $_\n");
			    exit;
			}
			$signify_pubkey .= '.pub' if (substr ($signify_pubkey, length ($signify_pubkey) - 4, 4)) ne '.pub';
			if (!-e "$SIGNIFY_DIR/$signify_pubkey") {
			    send_error ($config_file, "Signify public key doesn't exist, \"$signify_pubkey\" on line $line_num. $_\n");
			    exit;
			}
			if (!-r "$SIGNIFY_DIR/$signify_pubkey") {
			    send_error ($config_file, "Signify public key is not readable, \"$signify_pubkey\" on line $line_num. $_\n");
			    exit;	
			}
			if (!$have_signify) {
			    send_error ($config_file, "Signify.pm not found on system, but signify  public key is defined, \"$signify_pubkey\" on line $line_num. $_\n");
			    exit;	
			}
			$signify_pubkey = $SIGNIFY_DIR . '/' . $signify_pubkey;
		    }
		    elsif ($directive eq 'define_time') {
			my ($name, $spec) = split (/\s*=\s*/, $value, 2);

			# Parse the spec and immediately evaluate against current time.
			my $constraint = parse_time_spec ($spec);
			if (!defined($constraint)) {
			    send_error($config_file, "Invalid time specification on line $line_num: $spec\nHours must be 00-23, minutes must be 00-59. $_\n");
			    exit;
			}
			my $is_applicable = evaluate_time_constraint ($constraint, time());

			# Store just the boolean result
			$time_ranges{$name} = $is_applicable;

			if ($debug_mode) {
			    my $status = $is_applicable ? "ACTIVE" : "INACTIVE";
			    print "DEBUG: define_time: $name = $spec [$status]\n";
			}
		    }
		    elsif ($directive eq 'log') {
			# No longer check for logfile existence here due to need
			# to unveil first.
			# First, see if we already have this log file.
			if (grep ($_ eq $value, @logfiles)) {
			    if ($config_check && $multihost_config && !$oldstyle_multihost) {
				my (@hosts, $host);
				@hosts = split (/\s+/, $current_host);
				foreach $host (@hosts) {
				    if ($defined_hostlog{"$host:$value"}) {
					send_error ($config_file, "Previously defined logfile for host $host on line $line_num. $value");
					exit;
				    }
				    $defined_hostlog{"$host:$value"} = 1;
				}
				# don't push to @logfiles, it's already there
				$current_logfile = $value;
				next;
			    }
			    else {
				send_error ($config_file, "Previously defined logfile on line $line_num. $value");
				exit;
			    }
			}
			# Do minimal validation on log name.
			if (!valid_logfile_format ($value)) {
			    send_error ($config_file, "Invalid logfile name on line $line_num. $value");
			    exit;
			}
			if ($value =~ /^journal (unit|syslog-id|syslog-facility)/) {
			    # Don't validate the journal info if we're doing a config check unless we're on the right host.
			    if (!$config_check || my_host ($line_num, $current_host)) {
				if ($1 eq 'unit') {
				    $linux_journal_unit = 1;
				}
				elsif ($1 eq 'syslog-id') {
				    $linux_journal_syslogid = 1;
				}
				else {
				    $linux_journal_syslogfacility = 1;
				}
			    }
			}
			push (@logfiles, $value);
			$current_logfile = $value;
		    }
		    elsif ($directive eq 'match') {
			if (!defined ($current_logfile)) {
			    send_error ($config_file, "No log directive corresponding to match directive on line $line_num. $_");
			    exit;
			}
			if (($value eq 'all') || valid_regexp ($value) || ($value =~ /^session-with (.*$)/ && valid_regexp ($1))) {
			    $rule_text = $value;
			    $current_match_session_match_flag = 0;
			    if ($value eq 'all') {
				$rule = { type => 'all', text => 'all' };
			    }
			    else {
				# session-with
				if ($value =~ /^session-with (.*$)/) {
				    $value = $1;
				    $current_match_session_match_flag = 1;
				    $rule_type = 'session_with';
				}
				else {
				    $rule_type = 're';
				}
				# Do preproc macro substitution.
				$value = preproc_macro_substitution ($value, $line_num) if ($value =~ /%%[\w\-]+%%/);
				# Negate if it begins with ! and remove it.
				my $neg = ($value =~ s{^!}{});
				# Remove leading and trailing slash.
				$value =~ s/^\///;
				$value =~ s/\/$//;
				# Precompile regexp.
				my $re = compile_re ($value, "match rule on line $line_num. $_");
				$rule = {
				    type => $rule_type,
				    re => $re,
				    neg => $neg,
				    text => $rule_text
				}
			    }
			    push (@{$match_hash{$current_logfile}}, $rule);
			}
			else {
			    send_error ($config_file, "Invalid match directive on line $line_num. $_");
			    exit;
			}
		    }
		    elsif ($directive eq 'exclude') {
			if (!defined ($current_logfile)) {
			    send_error ($config_file, "No log directive corresponding to exclude directive on line $line_num. $_");
			    exit;
			}
			if (($value eq 'none') || valid_regexp ($value) || ($value =~ /^session-without (.*$)/ && valid_regexp ($1))) {
			    $rule_text = $value;
			    if ($value eq 'none') {
				$rule = { type => 'none', text => 'none' };
			    }
			    else {
				if ($value =~ /^session-without (.*$)/) {
				    $value = $1;
				    $rule_type = 'session_without';
				    if (!$current_match_session_match_flag) {
					send_error ($config_file, "Exclude directive is a session match but corresponding match directive is not on line $line_num. $_");
					exit;
				    }
				}
				else {
				    $rule_type = 're';
				    if ($current_match_session_match_flag) {
					send_error ($config_file, "Exclude directive is not a session match but corresponding match directive is on line $line_num. $_");
					exit;
				    }
				}
				# Do preproc macro substitution.
				$value = preproc_macro_substitution ($value, $line_num) if ($value =~ /%%[\w\-]+%%/);
				# Negate if it begins with ! and remove it.
				my $neg = ($value =~ s{^!}{});
				# Remove leading and trailing slash.
				$value =~ s/^\///;
				$value =~ s/\/$//;
				# Precompile regexp.
				my $re = compile_re ($value, "exclude rule on line $line_num. $_");
				$rule = {
				    type => $rule_type,
				    re => $re,
				    neg => $neg,
				    text => $rule_text
				};
			    }
			    push (@{$exclude_hash{$current_logfile}}, $rule);
			}
			else {
			    send_error ($config_file, "Invalid exclude directive on line $line_num. $_");
			}
		    }
		    elsif ($directive eq 'times') {
			    if (!defined($current_logfile)) {
				send_error($config_file, "No log directive corresponding to times directive on line $line_num. $_");
				exit;
			    }
    
			    my $num_matches = defined($match_hash{$current_logfile}) ? scalar(@{$match_hash{$current_logfile}}) : 0;
			    my $num_excludes = defined($exclude_hash{$current_logfile}) ? scalar(@{$exclude_hash{$current_logfile}}) : 0;
    
			    if ($num_matches == 0) {
				send_error ($config_file, "times directive on line $line_num must come after match directive");
				exit;
			    }
			    if ($num_excludes != $num_matches) {
				send_error ($config_file, "times directive on line $line_num must come after exclude directive (use 'exclude: none' if no exclusions)");
				exit;
			    }

			    # Evaluate the time constraint
			    my $negate = 0;
			    my $range_name = $value;

			    if ($value =~ /^!(.+)$/) {
				$negate = 1;
				$range_name = $1;
			    }

			    if (!exists($time_ranges{$range_name})) {
				send_error($config_file, "Undefined time range '$range_name' on line $line_num");
				exit;
			    }

			    my $applicable = $negate ? !$time_ranges{$range_name} : $time_ranges{$range_name};
    
			    # Store in parallel hash
			    push(@{$time_hash{$current_logfile}}, $applicable);

			    if ($debug_mode && !$applicable) {
				print "DEBUG: Line $line_num: times: $value -> rule will be skipped.\n";
			    }
		    }
		    elsif ($directive eq 'action') {
			if (!defined ($current_logfile)) {
			    send_error ($config_file, "No log directive corresponding to action directive on line $line_num. $_");
			    exit;
			}

			# Validate that we have matching match and exclude directives
			my $num_matches = defined($match_hash{$current_logfile}) ? scalar(@{$match_hash{$current_logfile}}) : 0;
			my $num_excludes = defined($exclude_hash{$current_logfile}) ? scalar(@{$exclude_hash{$current_logfile}}) : 0;
    
			if ($num_matches == 0) {
			    send_error ($config_file, "action directive on line $line_num has no corresponding match directive");
			    exit;
			}
			if ($num_excludes != $num_matches) {
			    send_error ($config_file, "action directive on line $line_num: mismatch between number of match ($num_matches) and exclude ($num_excludes) directives. Each match must have a corresponding exclude (use 'exclude: none' if no exclusions needed)");
			    exit;
			}
			# Check for semicolon (multiple actions).
			# Syntax: "action: notify email@example.com; execute script.sh"
			# Only execute permitted as second action, and only after notify/alert/text.
			my @action_specs;
			if ($value =~ /^([^;]+);(.*)$/) { # multiple
			    @action_specs = ($1, $2);
			    # Trim whitespace
			    $action_specs[0] =~ s/^\s+|\s+$//g;
			    $action_specs[1] =~ s/^\s+|\s+$//g;
			}
			else { # just one
			    @action_specs = ($value);
			}
			my @parsed_actions;
			for (my $idx = 0; $idx <= $#action_specs; $idx++) {
			    my $action_spec = $action_specs[$idx];
			    my ($action, $action_value);
			    # Parse action (action, whitespace, value).
			    if ($action_spec !~ /\s/) {
				$action = $value;
				undef $action_value;
			    }
			    else {
				($action, $action_value) = split (/\s+/, $action_spec, 2);
			    }

			    # Validate second action must be execute.
			    if ($idx == 1 && $action ne 'execute') {
				send_error ($config_file, "Second action after semicolon must be \"execute\" on line $line_num. $_");
				exit;
			    }

			    # Validate first action cannot be execute if there's a second.
			    if ($idx == 0 && @action_specs == 2 && $action eq 'execute') {
				send_error ($config_file, "Cannot have \"execute\" as first action when using semicolon syntax on line $line_num. $_");
				exit;
			    }

			    # Parse and validate each action type
			    # action: notify
			    if ($action eq 'notify') {
				if (defined ($action_value)) {
				    # Minimal email validation.
				    my @check_emails = split (/,\s*/, $action_value);
				    foreach my $check_email (@check_emails) {
					if (!valid_email ($check_email)) {
					    send_error ($config_file, "Invalid email address(es) following \"notify\" action in action directive on line $line_num. $_");
					    exit;
					}
				    }
				    push (@parsed_actions, "$action,$action_value");
				}
				else {
				    send_error ($config_file, "Missing email address(es) following \"notify\" action in action directive on line $line_num. $_");
				    exit;
				}
			    }
			    # action: text
			    elsif ($action eq 'text') {
				if (defined ($action_value)) {
				    # Validate email format at config parse time.
				    my @check_emails = split (/,\s*/, $action_value);
				    foreach my $check_email (@check_emails) {
					if (!valid_email ($check_email)) {
					    send_error ($config_file, "Invalid email address(es) following \"text\" action in action directive on line $line_num. $_");
					    exit;
					}
				    }
				    push (@parsed_actions, "$action,$action_value");

				}
				else {
				    send_error ($config_file, "Missing email address(es) following \"text\" action in action directive on line $line_num. $_");
				    exit;
				}
			    }
			    # action: alert
			    elsif ($action eq 'alert') {
				if (defined ($action_value)) {
				    send_error ($config_file, "Extraneous data following \"alert\" action in action directive on line $line_num. $_");
				    exit;
				}
				push (@parsed_actions, $action);
			    }
			    # action: execute
			    elsif ($action eq 'execute') {
				if (defined ($action_value)) {
				    # Validate script name (no path traversal, alphanumeric/underscore/hyphen/dot only)
				    if ($action_value !~ /^[\w\-\.]+$/) {
					send_error ($config_file, "Invalid script name \"$action_value\" following \"execute\" action in action directive on line $line_num. $_");
					exit;
				    }

				    # Construct full script path
				    my $script_path = "$SCRIPT_DIR/$action_value";

				    # Don't do these checks if we're doing a config check (-c) unless it's the current host.
				    if (!$config_check || my_host ($line_num, $current_host)) {
					# Verify script exists and is readable
					if (!-r $script_path) {
					    send_error ($config_file, "Cannot read execute script \"$script_path\" specified on line $line_num. $_");
					    exit;
					}

					# Verify script is a regular file (not symlink or directory)
					if (!-f $script_path || -l $script_path) {
					    send_error ($config_file, "Execute script \"$script_path\" must be a regular file (not symlink or directory) on line $line_num. $_");
					    exit;
					}

					# Verify signify signature exists and is valid
					if (!defined ($signify_pubkey)) {
					    send_error ($config_file, "No signify_pubkey directive has been parsed yet in config file before use of execute script \"$script_path\" on line $line_num. $_");
					    exit;
					}
					if (!$have_signify) {
					    send_error ($config_file, "Signify.pm not found on system, cannot verify signature on execute script \"$script_path\" on line $line_num. $_");
					    exit;	
					}
					if (!verify_signify_sig ($script_path)) {
					    send_error ($config_file, "Cannot verify signify signature on execute script \"$script_path\" on line $line_num. $_");
					    exit;
					}
				    }
				
				    # Track this script for later verification and unveiling
				    $execute_scripts{$script_path} = 1;
				    push (@parsed_actions, "$action,$action_value");
				}
				else {
				    send_error ($config_file, "Missing script name following \"execute\" action in action directive on line $line_num. $_");
				    exit;
				}
			    }
			    # Other actions?
			    else {
				send_error ($config_file, "Unknown action \"$action\" specified in action directive on line $line_num. $_");
				exit;
			    }
			}

			# @parsed_actions is already an array of strings, make it an array ref.
			push (@{$action_hash{$current_logfile}}, [@parsed_actions]);

			# After storing action, ensure time_hash is aligned
			# Check if time constraint was added for this triplet
			my $num_actions = scalar(@{$action_hash{$current_logfile}});
			my $num_times = defined($time_hash{$current_logfile}) ? scalar(@{$time_hash{$current_logfile}}) : 0;
    
			if ($num_times < $num_actions) {
			    # No times: directive for this triplet - add undef placeholder
			    push(@{$time_hash{$current_logfile}}, undef);
			}
		    }
		    # For backwards compatibility.
		    elsif ($directive eq 'notify') {
			if (!defined ($current_logfile)) {
			    send_error ($config_file, "No log directive corresponding to notify directive on line $line_num. $_");
			    exit;
			}
			# Store as array ref for consistency with action: directive
			push (@{$action_hash{$current_logfile}}, ["$directive,$value"]);
			# need to specify that action=notify, and add
			# separate code to parse new action: directive.
		    }
		    # For including a file of macro definitions.
		    elsif ($directive eq 'include-macro-file' ||
			   $directive eq 'include-macro-signedfile') {
			$macro_file = $value;
			if ($macro_file !~  /^[\w\-\.]+$/) {
			    send_error ($config_file, "Invalid macro include file name \"$macro_file\" in macro \"$macro_name\" on line $line_num. $_\n");
			    exit;
			}
			$macro_file = $MACRO_DIR . '/' . $macro_file;
			# Don't verify readability during config check if not current host.
			if (!$config_check || my_host ($line_num, $current_host)) {
			    if (!-r $macro_file) {
				send_error ($config_file, "Cannot read macro include file $macro_file in macro \"$macro_name\" on line $line_num. $_\n");
				exit;
			    }
			}
			if ($directive =~ /signed/) {
			    if (!defined ($signify_pubkey)) {
				send_error ($config_file, "No signify_pubkey directive has been parsed yet in config file before use of macro include file \"$macro_file\" on line $line_num. $_\n");
				exit;
			    }
			    # Don't verify presence of Signify.pm or signature during config check if not current host.
			    if (!$config_check || my_host ($line_num, $current_host)) {
				if (!$have_signify) {
				    send_error ($config_file, "Signify.pm not found on system, cannot verify signature on signed macro include file \"$macro_file\" on line $line_num. $_\n");
				    exit;	
				}
				if (!verify_signify_sig ($macro_file)) {
				    send_error ($config_file, "Cannot verify signify signature on signed macro include file \"$macro_file\" on line $line_num. $_\n");
				    exit;
				}
			    }
			}
			# Don't verify contents of macro file during config check if not current host, but warn about potential subsequent
			# missing macro definitions.
			if ($config_check && !my_host ($line_num, $current_host)) {
			    print "Note: Skipping macro file \"$macro_file\" on $line_num (applied to different host(s): $current_host)\n";
			    print "      Any macros defined in this file may show as undefined in subsequent warnings.\n";
			}
			elsif (open (INCLUDEFILE, '<', $macro_file)) {
			    my $include_file_line_num = 0;
			    while (<INCLUDEFILE>) {
				$include_file_line_num++;
				chomp;
				# ignore blank lines and comments
				if (!/^\s*$|^\s*#.*$/) {
				    if (/^([\w\-]+)\s*=\s*\"(.+)\"(.*$)/) {
					$macro_name = $1;
					$macro_value = $2;
					$macro_options = $3;
					parse_macro ($macro_name, $macro_value, $macro_options,
						      $current_context, $current_host,
						      $line_num, $include_file_line_num);
				    }
				    else {
					send_error ($config_file, "Invalid macro definition line in macro include file \"$macro_file\" line $include_file_line_num, config file line $line_num.\n");
				    }
				}
			    }
			} # open
			else {
			    send_error ($config_file, "Cannot open macro include file $macro_file in macro \"$macro_name\" on line $line_num. $! $_\n");
			}
		    }
		    else {
			send_error ($config_file, "Unknown directive on line $line_num. $_");
			exit;
		    }
		}
	    }
	}
	close (CONFIG);
    }
    else {
	die "Cannot open config file $config_file. $!\n";
	send_error ($config_file, "Cannot open config file. $!");
	exit;
    }

    # If multi-host, did we find our host in the config?
    if ($multihost_config && !$found_my_host) {
	send_error ($config_file, "Did not find this host ($HOSTNAME) in config file.");
	exit;
    }

    if (!defined ($size_file)) {
	$size_file = "$DEFAULT_SIZE_FILE_DIR/$DEFAULT_SIZE_FILE_NAME";
    }
    elsif (substr ($size_file, length ($size_file) - 5, 5) ne $SIZE_SUFFIX) {
	$size_file .= $SIZE_SUFFIX;
    }

    if (!defined ($master_notify)) {
	if (defined ($all_host_master_notify)) {
	    $master_notify = $all_host_master_notify;
	}
	else {
	    $master_notify = $SECURITY_ADMIN;
	}
    }

    if (!defined ($email_sender)) {
	$email_sender = $EMAIL_SENDER;
    }

    foreach $current_logfile (@logfiles) {
	# These checks need to be changed.
	if (!defined ($match_hash{$current_logfile})) {
	    my $rule = { type => 'all', text => 'all' };
	    push (@{$match_hash{$current_logfile}}, $rule);
	}
	if (!defined ($exclude_hash{$current_logfile})) {
	    my $rule = { type => 'none', text => 'none' };
	    push (@{$exclude_hash{$current_logfile}}, $rule);
	}
	if (!defined ($action_hash{$current_logfile})) {
	    # Default action is notify to master_notify (stored as array ref)
	    push (@{$action_hash{$current_logfile}}, [$master_notify]);
	    # need indicator for action=notify.
	}

	# Build *_hash_ref structures, filtering by time constraints
	my @applicable_matches;
	my @applicable_excludes;
	my @applicable_actions;

	my $actions_arrayref = $action_hash{$current_logfile};

	for (my $idx = 0; $idx < scalar(@{$actions_arrayref}); $idx++) {
	    # Check time constraint (if it exists)
	    if (defined($time_hash{$current_logfile}) && 
		defined($time_hash{$current_logfile}[$idx]) &&
		!$time_hash{$current_logfile}[$idx]) {
		# Time constraint not met - skip entire triplet
		next;
	    }
	    
	    # Include this triplet
	    push(@applicable_matches, $match_hash{$current_logfile}[$idx]);
	    push(@applicable_excludes, $exclude_hash{$current_logfile}[$idx]);
	    push(@applicable_actions, $actions_arrayref->[$idx]);
	}
    
	# Store references to the NEW arrays we just built
	# These are local @arrays, so just take normal references
	$match_hash_ref{$current_logfile} = \@applicable_matches;
	$exclude_hash_ref{$current_logfile} = \@applicable_excludes;
	$action_hash_ref{$current_logfile} = \@applicable_actions;
    }

    # If there are linux journal logs, pull validation data.
    if ($linux_journal_unit) {
	@linux_journal_units = get_linux_journal_units();
    }
    if ($linux_journal_syslogfacility) {
	@linux_journal_syslog_facilities = get_linux_journal_syslog_facilities();
    }
    if ($linux_journal_syslogid) {
	# don't currently validate.
    }
}

# Minimal validation on email address.  Better to use
# Mail::RFC822::Address's valid.
sub valid_email {
    my ($email) = @_;

    if ($email =~ /.+\@.+\..+/) {
	return 1;
    }

    return 0;
}

# Minimal validation on logfile name. Lots of room for improvement.
# Also not great to be checking the linux journal format in so many
# places.
sub valid_logfile_format {
    my ($logfile) = @_;

    if ($logfile =~ /\.\./) {
	return 0;
    }
    elsif ($logfile =~ /^\/[\w\-\.\/]+$/) {
	return 1;
    }
    elsif ($logfile =~ /^journal (unit|syslog-id|syslog-facility) ([\w\-\.]+)$/) {
	return 1;
    }

    return 0;
}


# Subroutine to get linux journal units. Only loaded active and with
# names ending in ".service".
# Doesn't require privileges.
sub get_linux_journal_units {
    my @units;

    if (open (SYSTEMCTL, '-|', $SYSTEMCTL, 'list-units')) {
	while (<SYSTEMCTL>) {
	    chomp;
	    if (/^\s*(.*\.service).*loaded active/) {
		push (@units, $1);
	    }
	}
	close (SYSTEMCTL);
    }
    else {
	send_error ($config_file, "Cannot pull journal units from $SYSTEMCTL. $!\n");
	exit;
    }
    return (@units);
}

# Subroutine to get available linux journal syslog facilities.
# Doesn't require privileges.
sub get_linux_journal_syslog_facilities {
    my @facilities;

    if (open (JOURNALCTL, '-|', $JOURNALCTL, '--facility=help')) {
	while (<JOURNALCTL>) {
	    chomp;
	    push (@facilities, $_) unless (/Available facilities:/);
	}
	close (JOURNALCTL);
    }
    else {
	send_error ($config_file, "Cannot pull syslog facilities from $JOURNALCTL. $!\n");
	exit;
    }
    return (@facilities);
}

# Macro parsing moved out to separate subroutine (poorly modularized)
# in order to add include-macro-file/include-macro-signedfile.
# $line_num is config line number.
sub parse_macro {
    my ($macro_name, $macro_value, $macro_options,
	$current_context, $current_host,
	$line_num, $include_file_line_num) = @_;
    my ($macro_file); # macro value file

    # Combine config line number and macro include file line number in errors
    # if we're processing a macro include file.
    if ($include_file_line_num) {
	$line_num = $include_file_line_num . ' of macro include file on config line ' . $line_num;
    }
    
    # if !defined ($current_host) then we're in global
    # context, otherwise we're in a specific host context.
    # need to convert all this stuff to subroutines.
    if (defined ($global_preproc_macro{$macro_name})) {
	send_error ($config_file, "Previously defined global preproc macro \"$macro_name\" on line $line_num. $_\n");
	exit;
    }
    if (defined ($preproc_macro{$macro_name})) {
	send_error ($config_file, "Previously defined host preproc macro \"$macro_name\" on line $line_num. $_\n");
	exit;
    }
    # Allow importation of macro value from a file in same
    # dir as config file. Only expand if either in global
    # context or if in host context for current host.
    if ($macro_value =~ /^<(?:signed)?file:(.*)>$/ &&
	(($current_context == $GLOBAL_CONTEXT) ||
	 ($current_context == $HOST_CONTEXT &&
	  $current_host eq $HOSTNAME))){
	$macro_file = $1;
	if ($macro_file !~  /^[\w\-\.]+$/) {
	    send_error ($config_file, "Invalid macro value file name \"$macro_file\" in macro \"$macro_name\" on line $line_num. $_\n");
	    exit;
	}
	$macro_file = $MACRO_DIR . '/' . $macro_file;
	if (!-r $macro_file) {
	    send_error ($config_file, "Cannot read macro value file $macro_file in macro \"$macro_name\" on line $line_num. $_\n");
	    exit;
	}
	if ($macro_value =~ /^signed/) {
	    if (!defined ($signify_pubkey)) {
		send_error ($config_file, "No signify_pubkey directive has been parsed yet in config file before use of macro value file \"$macro_file\" on line $line_num. $_\n");
		exit;			
	    }
	    if (!$have_signify) {
		send_error ($config_file, "Signify.pm not found on system, can't verify signature on signed macro value file \"$macro_file\" on line $line_num. $_\n");
		exit;	
	    }
	    if (!verify_signify_sig ($macro_file)) {
		send_error ($config_file, "Cannot verify signify signature on signed macro value file \"$macro_file\" on line $line_num. $_\n");
		exit;
	    }
	}
	if (open (VALUEFILE, '<', $macro_file)) {
	    $macro_value = "";
	    while (<VALUEFILE>) {
		chomp;
		# ignore blank lines and comments
		if (!/^\s*$|^\s*#.*$/) {
		    if ($macro_value eq "") {
			$macro_value = $_;
		    }
		    # If multiple lines, concatenate with |.
		    else {
			$macro_value .= '|' . $_;
		    }
		}
	    }
	    close (VALUEFILE);
	}
	else {
	    send_error ($config_file, "Cannot open macro value file $macro_file in macro \"$macro_name\" on line $line_num. $! $_\n");
	    exit;
	}
    }
    if ($current_context == $GLOBAL_CONTEXT) {
	$global_preproc_macro{$macro_name} = $macro_value;
    }
    else {
	$preproc_macro{$macro_name} = $macro_value;
    }
    if ($macro_options =~ /:(append|substitute)/) {
	if ($1 eq 'append') {
	    if (defined ($global_append_macro{$macro_value})) {
		send_error ($config_file, "Previously defined append global macro value \"$macro_value\" on line $line_num. $_\n");
		exit;				
	    }
	    if (defined ($append_macro{$macro_value})) {
		send_error ($config_file, "Previously defined append host macro value \"$macro_value\" on line $line_num. $_\n");
		exit;
	    }
	    if ($current_context == $GLOBAL_CONTEXT) {
		$global_append_macro{$macro_value} = $macro_name;
		$have_global_postproc_macros = 1;
	    }
	    else {
		$append_macro{$macro_value} = $macro_name;
		$have_postproc_macros = 1;
	    }
	}
	else {
	    if (defined ($global_substitute_macro{$macro_value})) {
		send_error ($config_file, "Previously defined substitute global macro value \"$macro_value\" on line $line_num. $_\n");
		exit;
	    }
	    if (defined ($substitute_macro{$macro_value})) {
		send_error ($config_file, "Previously defined substitute host macro value \"$macro_value\" on line $line_num. $_\n");
		exit;
	    }
	    if ($current_context == $GLOBAL_CONTEXT) {
		$global_substitute_macro{$macro_value} = $macro_name;
		$have_global_postproc_macros = 1;
	    }
	    else {
		$substitute_macro{$macro_value} = $macro_name;
		$have_postproc_macros = 1;
	    }
	}
    }
    elsif ($macro_options ne '') {
	send_error ($config_file, "Invalid macro options for macro \"$macro_name\" on line $line_num. $_\n");
	exit;
    }
}

# Verify signify signature on a file. Code originally derived from subroutine
# sigtree_signify_verify in sigtree.pl, now modified to use Signify.pm.
sub verify_signify_sig {
    my ($file) = @_;
    my (@errors);
    my $SKIP_SIGNIFY_CHECK = 0;
    my $SKIP_PRECHECKS = 0;

    # Already checked readability of file itself but Signify.pm will
    # check it again.
    if (Signify::verify ($file, $signify_pubkey)) {
	return 1;
    }

    @errors = Signify::signify_error();

    # Report any errors, apart from readability of file itself.
    if ($errors[0] =~ /^no executable/) {
	send_error ($config_file, "No signify binary on system to verify signify signature.\n");
	return 0;
    }
    elsif ($errors[0] =~ /^no readable signature file/) {
	send_error ($config_file, "Cannot open signed macro file \"$file\".\n");
	return 0;
    }
    elsif ($errors[0] =~ /^no readable public key/) {
	send_error ($config_file, "Cannot open signify public key \"$signify_pubkey\" to verify signify signature.\n");
	return 0;
    }
    # shouldn't happen without opportune deletion.
    elsif ($errors[0] =~ /^no readable file/) {
	send_error ($config_file, "Cannot read file $file to verify signify signature.\n");
	return 0;
    }
    else {
	send_error ($config_file, "@errors");
	return 0;
    }
}

# Subroutine to return 1 if a host list includes the current host or "all"
# (or "all except ..." without excluding the current host), and 0 otherwise.
sub my_host {
    my ($line_num, $host_list) = @_;
    my (@hosts, $host);

    # 'all'
    return 1 if ($host_list eq 'all');

    # 'all except'
    if ($host_list =~ /^all\s+except\s+(.*)$/) {
	my $except_list = $1;
	my @excluded_hosts = split (/\s+/, $except_list);

	# Validate excluded hosts if config checking
        if ($config_check) {
            foreach my $excluded (@excluded_hosts) {
                if (!defined($defined_hosts{$excluded})) {
                    print "New host name in \"hosts:\" directive exception list on line $line_num. $excluded\n";
                    $defined_hosts{$excluded} = 1;
                }
            }
        }
	# Check if current host is in the exception list
        return 0 if (grep { $_ eq $SHORT_HOSTNAME } @excluded_hosts);
        return 0 if (grep { $_ eq $HOSTNAME } @excluded_hosts);
        
        # Not in exception list, so "all except" includes this host
        return 1;
    }

    # Handle specific host list.
    @hosts = split (/\s+/, $host_list);
    if ($config_check) {
	foreach $host (@hosts) {
	    if (!defined ($defined_hosts{$host})) {
		print "New host name in \"hosts:\" directive on line $line_num. $host\n";
		$defined_hosts{$host} = 1;
	    }
	}
    }

    return 1 if (grep { $_ eq $SHORT_HOSTNAME } @hosts);
    return 1 if (grep { $_ eq $HOSTNAME } @hosts);
    return 0;
}

# Subroutine to return accounting clock tick rate.
sub accounting_hz {
    if ($^O eq 'linux') {
        return sysconf (_SC_CLK_TCK);
    }
    else {
        return 64; # BSD/macOS accounting clock tick rate.                                                                                              
    }
}

# Build %devname_cache on OpenBSD and macOS. This is done before
# unveiling for convenience (File::Find uses cd in a way that
# would require unveiling both /dev and the directory from which
# reportnew is being run). It doesn't require any privileges other
# than what is in the initial promises--but on macOS, root will
# build a more complete cache; it will return the same results
# as lastcomm (which on macOS has different tty output when run
# as a nonprivileged user vs. root--more ?? results).
sub build_devname_cache {
    my $DEVDIR = '/dev';

    find (sub {
        return unless -c $_; # only character devices
        my $fullpath = $File::Find::name;
        my @st = lstat ($fullpath);
        my $rdev = $st[6]; # st_rdev
	$devname_cache{$rdev} ||= basename ($fullpath);
          }, $DEVDIR);

    $devname_cache{-1} = '__';
}

## Subroutines for time specifications.
sub parse_time_spec {
    my ($spec) = @_;
    
    # First, expand any shorthand notation
    my @range_specs;
    if ($spec =~ /, /) {
        # Multiple ranges specified
        my @parts = split(/, /, $spec);
        foreach my $part (@parts) {
            my @expanded = expand_time_spec($part);
            return undef unless @expanded;  # Expansion failed
            push(@range_specs, @expanded);
        }
    } else {
        # Single range
        @range_specs = expand_time_spec($spec);
        return undef unless @range_specs;  # Expansion failed
    }
    
    # Now parse the expanded ranges
    if (@range_specs > 1) {
        # Multiple ranges - create OR constraint
        my @constraints;
        foreach my $range (@range_specs) {
            my $constraint = parse_single_time_spec($range);
            return undef unless defined($constraint);
            push(@constraints, $constraint);
        }
        return { type => 'OR', constraints => \@constraints };
    } else {
        # Single range
        return parse_single_time_spec($range_specs[0]);
    }
}

sub parse_single_time_spec {
    my ($spec) = @_;
    
    # Format: "Mon-Fri 09:00-17:00" or "daily 02:00-04:00" or "Sat-Sun all"
    if ($spec =~ /^([\w,-]+)\s+(all|\d{2}:\d{2}-\d{2}:\d{2})$/) {
        my ($day_spec, $time_spec) = ($1, $2);
        
        my %constraint;
        
        # Parse days
        $constraint{days} = parse_day_spec($day_spec);
        return undef unless defined $constraint{days};
        
        # Parse time range
        unless ($time_spec eq 'all') {
            if ($time_spec =~ /^(\d{2}):(\d{2})-(\d{2}):(\d{2})$/) {
		my ($h1, $m1, $h2, $m2) = ($1, $2, $3, $4);
                
                # Validate time values
                return undef if ($h1 > 23 || $h2 > 23 || $m1 > 59 || $m2 > 59);
                
                my $start_min = $h1 * 60 + $m1;
                my $end_min = $h2 * 60 + $m2;
                $constraint{time_range} = [$start_min, $end_min];
            }
	    else {
                return undef;  # Invalid time format
            }
        }
        
        return \%constraint;
    }
    
    return undef;  # Invalid format
}

sub parse_day_spec {
    my ($day_spec) = @_;
    
    my %day_map = (
        'Mon' => 1, 'Tue' => 2, 'Wed' => 3, 'Thu' => 4,
        'Fri' => 5, 'Sat' => 6, 'Sun' => 0
    );
    
    my @applicable_days = (0) x 7;  # Array of 7 booleans
    
    if ($day_spec eq 'daily') {
        @applicable_days = (1) x 7;
    }
    elsif ($day_spec =~ /^(\w+)-(\w+)$/) {
        # Range: Mon-Fri
        my $start = $day_map{$1};
        my $end = $day_map{$2};
        
        if (!defined($start) || !defined($end)) {
            return undef;  # Invalid day names
        }
        
        # Handle wrapping (e.g., Fri-Mon)
        if ($start <= $end) {
            for (my $d = $start; $d <= $end; $d++) {
                $applicable_days[$d] = 1;
            }
        } else {
            # Wraps around week
            for (my $d = $start; $d <= 6; $d++) {
                $applicable_days[$d] = 1;
            }
            for (my $d = 0; $d <= $end; $d++) {
                $applicable_days[$d] = 1;
            }
        }
    }
    elsif ($day_spec =~ /,/) {
        # Comma-separated: Mon,Wed,Fri
        my @days = split(/,/, $day_spec);
        foreach my $day (@days) {
            my $day_num = $day_map{$day};
            return undef unless defined($day_num);
            $applicable_days[$day_num] = 1;
        }
    }
    else {
        # Single day: Mon
        my $day_num = $day_map{$day_spec};
        return undef unless defined($day_num);
        $applicable_days[$day_num] = 1;
    }
    
    return \@applicable_days;
}

# *:MM-MM shorthand expansion
sub expand_time_spec {
    my ($spec) = @_;
    
    # Check for *:MM-MM shorthand (every hour at minute MM)
    if ($spec =~ /^([\w,-]+)\s+\*:(\d{2})-(\d{2})$/) {
        my ($day_spec, $start_min, $end_min) = ($1, $2, $3);
        
        # Validate minutes
        if ($start_min > 59 || $end_min > 59) {
            return undef;  # Invalid - caller will report error
        }
        
        # Expand to 24 ranges (one per hour)
        my @expanded;
        for (my $hour = 0; $hour < 24; $hour++) {
            my $range_start = sprintf("%02d:%02d", $hour, $start_min);
            my $range_end = sprintf("%02d:%02d", $hour, $end_min);
            push(@expanded, "$day_spec $range_start-$range_end");
        }
        
        return @expanded;
    }
    
    # No shorthand - return as-is
    return ($spec);
}

sub evaluate_time_constraint {
    my ($constraint, $timestamp) = @_;
    
    # If it's an OR of multiple constraints
    if ($constraint->{type} && $constraint->{type} eq 'OR') {
        # Match if ANY sub-constraint matches
        foreach my $sub_constraint (@{$constraint->{constraints}}) {
            return 1 if evaluate_time_constraint($sub_constraint, $timestamp);
        }
        return 0;  # None matched
    }
    
    # Single constraint
    my ($sec,$min,$hour,$mday,$mon,$year,$wday) = localtime($timestamp);
    
    # Check day of week
    if (defined($constraint->{days})) {
        return 0 unless $constraint->{days}[$wday];
    }
    
    # Check time range (if day matched or no day constraint)
    if (defined($constraint->{time_range})) {
        my $current_min = $hour * 60 + $min;
        my ($start_min, $end_min) = @{$constraint->{time_range}};
        
        # Handle overnight ranges (22:00-06:00)
        if ($start_min > $end_min) {
            return ($current_min >= $start_min || $current_min < $end_min);
        } else {
            return ($current_min >= $start_min && $current_min < $end_min);
        }
    }
    
    return 1;  # No constraints or all matched
}

## Subroutines for privilege separation.

# Subroutine for privileged process to listen for and process requests.
# Requests are submitted as signed JSON using a shared secret generated
# before the unprivileged process is forked off. The content of a request
# is a request name and a hash of parameters, plus a request_id and
# timestamp. The response is a hash of # status (success, failure, or fd_follows),
# request_id, timestamp, and data (and possibly metadata). Wrappers are provided
# for requests for file descriptor or other privileged data, and for responses,
# error responses, and pre-validation/no HMAC responess.
# Request types and their required parameters:
# get-fd, get-fd-raw, get-mclog-fd: logname (which can for all requests include [.N[.gz]]
# at the end if not cyclog/multilog); get-mclog-fd requires a separate filename parameter
# for the specific file in the multilog/cyclog directory to be opened. These return a file
# descriptor immediately after a non-failure (sending back fd_follows status).
# get-hash: logname. Returns hash of first line of logfile (uncompressed if it's a gzip).
# get-journal: logname, checktime. Returns hash of temp_logfile and new checktime.
# get-linux-pacct (deprecated): logname, checktime. Returns hash of temp_logfile and new checktime.
# get-mclog-files: logname. Returns array reference of multilog/cyclog file list.
# gunzip: lofname. Gunzips the file and returns temp_logfile name.
sub priv_listener {
    my ($req, $req_type);
    my $MAX_REQUEST_LENGTH = 1024 * 1024;

    print "DEBUG: starting priv_listener\n" if ($debug_mode);
    while (1) {
	read ($parent_sock, my $len_bytes, 4) or last;
	my $msg_len = unpack ('N', $len_bytes);
	if ($msg_len > $MAX_REQUEST_LENGTH || $msg_len < 1) {
	    warn "Invalid message length: $msg_len\n";
	    send_error_response_nohmac ("Invalid message length.");
	    next;
	}

	# Read full message.
	my $buffer = '';
	my $remaining = $msg_len;
	while ($remaining > 0) {
	    my $n = read($parent_sock, my $chunk, $remaining);
	    last unless defined($n) && $n > 0;
	    $buffer .= $chunk;
	    $remaining -= $n;
	}
	if ($remaining > 0) {
	    warn "Short read on message body\n";
	    send_error_response_nohmac("Short read");
	    next;
	}

	my $envelope = eval {decode_json ($buffer) };

	if ($@) {
	    warn "Failed to parse envelope JSON: $@\n";
	    send_error_response_nohmac ("Invalid JSON.");
	    next;
	}

	# Verify HMAC.
	my $request_json = $envelope->{data};
	my $received_hmac = $envelope->{hmac};
	my $expected_hmac = hmac_sha256_hex ($request_json, $HMAC_SECRET);

	# This is potentially subject to timing attacks since it is not a constant-time
	# comparison, but using HMACs is already overkill in this implementation.
	unless ($received_hmac eq $expected_hmac) {
	    warn "HMAC verification failed.\n";
	    send_error_response_nohmac ("HMAC verification failed.");
	    next;
	}

	# Parse request.
	my $request = eval { decode_json ($request_json) };
	
	if ($@ || !validate_request ($request)) {
	    warn "Invalid request format.\n";
	    send_error_response ("Invalid request.", $request->{request_id});
	    next;
	}

	print "DEBUG: parent received $request->{type} (id: $request->{request_id})\n" if ($debug_mode);

	# Reject stale or future requests (tight window).
	my $age = time() - $request->{timestamp};
	if ($age > 15 || $age < -5) { # allow 15 second old, 5 seconds future
	    warn "Request timestamp out of acceptable range: $age seconds\n";
	    send_error_response ("Request too old or timestamp invalid", $request->{request_id});
	    next;
	}

	# Replay protection: reject duplicate request_id.
	state %seen_request_ids;
	if (exists $seen_request_ids{$request->{request_id}}) {
	    send_error_response ("Replay detected for request_id", $request->{request_id});
	    next;
	}

	# Record current request_id.
	$seen_request_ids{$request->{request_id}} = time();

	# Prune old request_ids.
	for my $rid (keys (%seen_request_ids)) {
	    delete $seen_request_ids{$rid}
	    if $seen_request_ids{$rid} < time() - 30;
	}

	# Process based on request type.
	if ($request->{type} eq 'get-fd' ||
	    $request->{type} eq 'get-fd-raw' ||
	    $request->{type} eq 'get-mclog-fd') {

	    handle_fd_request ($request);
	}
	elsif ($request->{type} eq 'get-hash') {
	    handle_hash_request ($request);
	}
	elsif ($request->{type} eq 'get-journal') {
	    handle_journal_request ($request);
	}
	elsif ($request->{type} eq 'get-linux-pacct') {
	    handle_linux_pacct_request ($request);
	}
	elsif ($request->{type} eq 'get-mclog-files') {
	    handle_mclog_files_request ($request);
	}
	elsif ($request->{type} eq 'gunzip') {
	    handle_gunzip_request ($request);
	}
	else {
	    send_error_response ("Unknown request type.", $request->{request_id});
	}
    }
}

# Subroutine to handle file descriptor requests (privileged side).
sub handle_fd_request {
    my ($request) = @_;

    my $logfile = $request->{params}{logfile};
    my $open_mode = $request->{params}{raw_mode} ? '<:raw' : '<';

    # Validate that logfile is in allowed list.
    unless (is_allowed_logfile ($logfile)) {
	send_error_response ("Logfile not in allowed list", $request->{request_id});
	return;
    }

    # Special case for cyclog.
    if ($request->{type} eq 'get-mclog-fd') {
	$logfile = $logfile . '/' . $request->{params}{filename};
    }

    # Path traversal check.
    if ($logfile =~ /\.\./) {
	send_error_response ("Path traversal attempt", $request->{request_id});
	return;
    }

    # Try to open the file.
    if (open (my $log_fh, $open_mode, $logfile)) {
	# Send response indicating that file descriptor will follow.
	my $response = {
	    status => RESP_FD_FOLLOWS,
	    request_id => $request->{request_id},
	    metadata => {
		logfile => $logfile,
		mode => $open_mode,
	    }
	};

	send_response ($response);

	# ACK protocol only on Linux to avoid FD passing race conditions.
	if ($^O eq 'linux') {
	    print "DEBUG: parent waiting for ACK before sending FD\n" if ($debug_mode);

	    my $ack_buf;
	    # Timeout to avoid hanging forever
	    eval {
		local $SIG{ALRM} = sub { die "ACK timeout\n"; };
		alarm (5); # 5 second timeout
		my $n = sysread ($parent_sock, $ack_buf, 3);
		alarm (0);

		unless (defined ($n) && $n == 3 && $ack_buf eq 'ACK') {
		    warn "Failed to receive ACK for FD transfer for $logfile (got \"$ack_buf\", " .
			(defined ($n) ? "$n bytes" : "undef") . ")\n";
		    close $log_fh;
		    send_error_response ("Failed to receive ACK for FD transfer", $request->{request_id});
		    return;
		}
	    };
	    if ($@) {
		warn "ACK timeout or error for $logfile: $@\n";
		close $log_fh;
		send_error_response ("ACK timeout: $@", $request->{request_id});
		return;
	    }
	    print "DEBUG: parent received ACK, sending FD\n" if ($debug_mode);
	}

	# Now send the file descriptor.
	IO::FDPass::send (fileno ($parent_sock), fileno ($log_fh))
	    or warn "Failed to send file descriptor for $logfile: $!\n";

	close ($log_fh); # safe to close after send
    }
    else {
	send_error_response ("Cannot open $logfile: $!", $request->{request_id}, int ($!));
    }
}

# Subroutine to handle hash requests (privileged side).
# Not used for (and doesn't check allowed list properly) for cyclog/multilog.
# If it becomes necessary, add get-mclog-hash and pass $cyclog_flag to
# is_allowed_logfile.
sub handle_hash_request {
    my ($request) = @_;

    my $logfile = $request->{params}{logfile};

    unless (is_allowed_logfile ($logfile)) {
	send_error_response ("Logfile not in allowed list.", $request->{request_id});
	return;
    }

    my $hash = eval { first_log_line_sha256_digest ($logfile) };
    
    if ($@) {
        send_error_response ("Error computing hash for $logfile: $@", $request->{request_id});
        return;
    }
    if (!defined ($hash) || $hash eq '' || $hash eq '0') {
        send_error_response ("Could not compute hash for $logfile.", $request->{request_id});
        return;
    }

    my $response = {
	status => RESP_SUCCESS,
	request_id => $request->{request_id},
	data => $hash,
    };

    send_response ($response);
}

# Subroutine to get Linux journal data (privileged side).
sub handle_journal_request {
    my ($request) = @_;

    my $logfile = $request->{params}{logfile};
    my $checktime = $request->{params}{checktime};

    unless (is_allowed_logfile ($logfile)) {
	send_error_response ("Logfile not allowed.", $request->{request_id});
	return;
    }

    my ($temp_logfile, $new_checktime) = read_linux_journal ($logfile, $checktime, $temp_dir);

    if (!defined ($temp_logfile)) {
        send_error_response ("Could not read journal for $logfile.", $request->{request_id});
        return;
    }
    
    chown ($reporter_uid, $reporter_gid, $temp_logfile);

    my $response = {
	status => RESP_SUCCESS,
	request_id => $request->{request_id},
	data => {
	    temp_logfile => $temp_logfile,
	    checktime => $new_checktime,
	}
    };

    send_response ($response);
}

# Get Linux process accounting logs (using lastcomm, deprecated; privileged side).
sub handle_linux_pacct_request {
    my ($request) = @_;

    my $logfile = $request->{params}{logfile};
    my $checktime = $request->{params}{checktime};
    
    unless (is_allowed_logfile ($logfile)) {
	send_error_response ("Logfile not allowed.", $request->{request_id});
	return;
    }

    my ($temp_logfile, $new_checktime) = read_process_acct_log ($logfile, $checktime, $temp_dir);
    
    if (!defined ($temp_logfile)) {
	send_error_response ("Could not read process accounting log for $logfile.", $request->{request_id});
	return;
    }
    
    chown ($reporter_uid, $reporter_gid, $temp_logfile);

    my $response = {
	status => RESP_SUCCESS,
	request_id => $request->{request_id},
	data => {
	    temp_logfile => $temp_logfile,
	    checktime => $new_checktime,
	}
    };

    send_response ($response);
}

# Get files in a multilog/cyclog directory (privileged side).
sub handle_mclog_files_request {
    my ($request) = @_;

    my $logfile = $request->{params}{logfile};

    unless (is_allowed_logfile ($logfile)) {
	send_error_response ("Logfile not allowed.", $request->{request_id});
	return;
    }

    $! = 0; # clear errno before call
    my @cyclog_files = get_cyclog_files ($logfile);

    if ($!) {
	send_error_response ("Could not open cyclog/multilog directory for $logfile: $!", $request->{request_id});
	return;
    }
    if (!@cyclog_files) {
	send_error_response ("Empty cyclog/multilog directory for $logfile.", $request->{request_id});
	return;
    }

    my $response = {
	status => RESP_SUCCESS,
	request_id => $request->{request_id},
	data => \@cyclog_files,
    };

    send_response ($response);
}

# Subroutine to handle gunzip (privileged side).
sub handle_gunzip_request {
    my ($request) = @_;

    my $logfile = $request->{params}{logfile};

    unless (is_allowed_logfile ($logfile)) {
	send_error_response ("Logfile not allowed.", $request->{request_id});
	return;
    }

    my $temp_logfile = eval { gunzip_logfile ($logfile, $temp_dir) };

    if ($@) {
	send_error_response ("Error gunzipping $logfile: $@", $request->{request_id});
	return;
    }
    if (!defined ($temp_logfile)) {
	send_error_response ("Could not gunzip $logfile.", $request->{request_id});
	return;
    }
    
    chown ($reporter_uid, $reporter_gid, $temp_logfile);

    my $response = {
	status => RESP_SUCCESS,
	request_id => $request->{request_id},
	data => $temp_logfile,
    };

    send_response ($response);
}

# Subroutine to send response from priv to nonpriv.
sub send_response {
    my ($response) = @_;

    my $response_json = encode_json ($response);
    my $hmac = hmac_sha256_hex ($response_json, $HMAC_SECRET);

    my $envelope = {
	data => $response_json,
	hmac => $hmac,
    };

    my $envelope_json = encode_json ($envelope);
    my $len = pack ('N', length ($envelope_json));
    print $parent_sock $len . $envelope_json;
}

# Subroutine to send an error response from priv to nonpriv.
sub send_error_response {
    my ($error_msg, $request_id, $errno) = @_;

    my $response = {
	status => RESP_ERROR,
	request_id => $request_id,
	error => $error_msg,
    };

    $response->{errno} = $errno if (defined ($errno));

    send_response ($response);
}

# For errors before we parse request_id (like HMAC failure),
# from priv to nonpriv.
sub send_error_response_nohmac {
    my ($error_msg) = @_;

    my $response = {
	status => RESP_ERROR,
	error => $error_msg,
    };

    # Still send with HMAC even if we couldn't verify incoming HMAC,
    # this allows legitimate clients to verify our response.
    send_response ($response);
}

# Subroutine to validate request format (privileged side).
sub validate_request {
    my ($request) = @_;
    my $REQUEST_ID_MAX_LENGTH = 128;

    return 0 unless ref ($request) eq 'HASH';
    return 0 unless defined ($request->{version}) && $request->{version} =~ /^\d+$/ && $request->{version} == 1;
    return 0 unless exists $request->{type};
    return 0 unless ref ($request->{params}) eq 'HASH';
    return 0 unless exists $request->{request_id};
    return 0 unless exists $request->{timestamp};

    # Validate request_id but allowing for future format changes
    return 0 unless length ($request->{request_id}) <= $REQUEST_ID_MAX_LENGTH;
    return 0 unless $request->{request_id} =~ /^[\w\.-]+$/a;

    # Validate timestamp is a number.
    return 0 unless $request->{timestamp} =~ /^\d+$/;

    # Check for path traversal or nulls in logfile name.
    if ($request->{params}{logfile} &&
	($request->{params}{logfile} =~ /\.\./ || $request->{params}{logfile} =~ /\0/)) {
	return 0;
    }

    return 1;
}

# Subroutine to verify that logfile is in permitted list from config (privileged side).
sub is_allowed_logfile {
    my ($logfile, $cyclog_file_flag) = @_;

    #   For rotated and gzipped logs, strip off those suffixes.
    if ($logfile =~ /^(.*)(\.\d(\.gz){0,1})$/) {
	$logfile = $1;
    }

    return grep { $_ eq $logfile } @logfiles;
}

# Subroutine to generate a request_id (nonprivileged side).
sub generate_request_id {
    state $counter = 0;
    my $rand = unpack ("H8", pack ("N", rand(0xffffffff)));
    return sprintf ("%d.%d.%s.%d", $$, time(), $rand, $counter++);
}

# Subroutine to generate shared secret for HMAC (pre-privilege separation).
sub generate_random_secret {
    my $entropy = join (':',
			$$,
			time(),
			rand(),
			$^T,
			$<,
			$>,
			$ENV{RANDOM_SEED} || '',
	);

    if (open (my $urandom, '<:raw', '/dev/urandom')) {
	my $n = read ($urandom, my $random_bytes, 32);
	close ($urandom);
	die "Could not get 32 bytes of randomness from /dev/urandom for HMAC secret generation.\n" if (!defined ($n) || $n != 32);
	$entropy .= unpack ('H*', $random_bytes);
    }
    else {
	die "Could not open /dev/urandom for HMAC secret generation. $!\n";
    }

    # Hash it all together for shared secret between priv and nonpriv process.
    return sha256_hex ($entropy);
}

# Subroutine to send a request to the privileged parent process and
# get a response, including for file descriptor requests.
sub send_priv_req {
    my ($req_type, $req_params) = @_;
    my $MAX_RESPONSE_LENGTH = 1024 * 1024;

    if (!$privsep_flag) {
	die "Call to send_priv_req when not using privilege separation.\n";
    }
    elsif ($priv_flag) {
	die "Call to send_priv_req from privileged parent process.\n";
    }

    my $request = {
	version => 1,
	type => $req_type,
	params => $req_params,
	request_id => generate_request_id(),
	timestamp => time(),
    };

    # Serialize and sign.
    my $request_json = encode_json ($request);
    my $hmac = hmac_sha256_hex ($request_json, $HMAC_SECRET);

    my $envelope = {
	data => $request_json,
	hmac => $hmac,
    };

    my $envelope_json = encode_json ($envelope);
    my $len = pack ('N', length ($envelope_json));

    print "DEBUG: child sending $req_type\n" if ($debug_mode);
    print $child_sock $len . $envelope_json;

    # Read response.
    read ($child_sock, my $resp_len_bytes, 4)
	or die "Failed to read response length after sending priv request.\n";
    my $resp_len = unpack ('N', $resp_len_bytes);

    die "Response to priv request too large.\n" if $resp_len > $MAX_RESPONSE_LENGTH;

    read ($child_sock, my $resp_json, $resp_len)
	or die "Failed to read response after sending priv request.\n";

    # Verify response HMAC.
    my $resp_envelope = decode_json ($resp_json);
    my $resp_data = $resp_envelope->{data};
    my $resp_hmac = $resp_envelope->{hmac};
    my $expected_hmac = hmac_sha256_hex ($resp_data, $HMAC_SECRET);

    # This is potentially subject to timing attacks since it is not a constant-time
    # comparison, but using HMACs is already overkill in this implementation.
    unless ($resp_hmac eq $expected_hmac) {
	die "Response HMAC verification failed!\n";
    }
    
    my $response = decode_json ($resp_data);

    print "DEBUG: child received response: $response->{status} (id: $response->{request_id})\n" if ($debug_mode);

    # Verify request_id matches.
    if ($response->{request_id} ne $request->{request_id}) {
	die "Response request_id mismatch! Sent: $request->{request_id}, Received: $response->{request_id}\n";
    }

    return $response;
}

# Wrapper for file descriptor requests (nonprivileged side).
sub send_priv_req_for_fd {
    my ($req_type, $req_params) = @_;

    my $logfile = $req_params->{logfile};

    my $response = send_priv_req ($req_type, $req_params);
    
    if ($response->{status} eq RESP_FD_FOLLOWS) {
	print "DEBUG: receiving file descriptor\n" if ($debug_mode);

	# On Linux, send ACK before calling recv to avoid race condition
	if ($^O eq 'linux') {
	    print "DEBUG: child sending ACK before recv\n" if ($debug_mode);
	    syswrite ($child_sock, 'ACK', 3);
	}
	
	my $fd = IO::FDPass::recv (fileno ($child_sock));
	print "DEBUG: child received fd for $logfile\n" if ($debug_mode);

	if ($fd < 0) {
	    die "Failed to receive file descriptor.\n";
	}

	print "DEBUG: child opening fh for $logfile\n" if ($debug_mode);
	open (my $fh, "+<&=$fd")
	    or die "Cannot fdopen received file descriptor for $logfile: $!\n";
	return $fh;
    }
    elsif ($response->{status} eq RESP_ERROR) {
	$! = $response->{errno} if (defined ($response->{errno}));
	return;
    }
    else {
	die "Unexpected response: $response->{status}\n";
    }
}

# Wrapper for data requests (non-file descriptors; nonprivileged side).
sub get_priv_data {
    my ($req_type, $req_params) = @_;

    my $response = send_priv_req ($req_type, $req_params);

    if ($response->{status} eq RESP_SUCCESS) {
	return $response->{data};
    }
    elsif ($response->{status} eq RESP_ERROR) {
	die "Privileged request $req_type failed for $req_params->{logfile}: $response->{error}\n";
    }
    else {
	die "Unexpected response to $req_type request for $req_params->{logfile}: $response->{status}\n";
    }
}

# Subroutine as wrapper for file open (nonprivileged side). Open if readable, send priv request
# if using privilege separation and running unprivileged.
# Tries the open instead of checking -r so that $! gets returned with
# an appropriate error message.
sub nonpriv_open_fh {
    my ($logfile, $cyclog_file_flag, $raw_mode_flag) = @_;
    my ($log_fd, $log_fh, $mclog_dir, $mclogfile, $success_or_failure);

    my $open_mode = $raw_mode_flag ? '<:raw' : '<';

    print "DEBUG: nonpriv_open_fh on $logfile\n" if ($debug_mode);

    return $log_fh if (open ($log_fh, $open_mode, $logfile));

    if ($privsep_flag && $nonpriv_flag) {
	print "DEBUG: child sending priv req for fd for $logfile\n" if ($debug_mode);

	my ($req_type, $params);

	if ($cyclog_file_flag) {
	    my $mclog_dir = dirname ($logfile);
	    my $mclogfile = basename ($logfile);
	    $req_type = 'get-mclog-fd';
	    $params = {
		logfile => $mclog_dir,
		filename => $mclogfile,
	    };
	}
	else {
	    $req_type = $raw_mode_flag ? 'get-fd-raw' : 'get-fd';
	    $params = {
		logfile => $logfile,
	    };
	}

	return send_priv_req_for_fd ($req_type, $params);
    }

    return 0;
}

# Subroutine to get cyclog files in dir (nonprivileged).
# Could require privileges, but we try once without even with
# privilege separation.
sub get_cyclog_files {
    my ($logfile) = @_;
    my (@cyclog_files);

    if (opendir (CYCLOG, $logfile)) {
	@cyclog_files = grep (!/^\./, readdir (CYCLOG));
	closedir (CYCLOG);
	return (@cyclog_files);
    }
    elsif ($privsep_flag && $nonpriv_flag) {
	my $files_ref = get_priv_data ('get-mclogfiles', { logfile => $logfile });
	@cyclog_files = @$files_ref;
	return (@cyclog_files);
    }

    # Failed.
    return ();
}

## Main log checker.

# Look through a log file for any changed lines; add them to
# the global variable @notify_lines and execute the corresponding action.
#
# If $log_type == $LOG_TYPE_PROCESS_ACCOUNTING, then the display and
# size file reference is different from the name of the file actually
# being checked. (Should this be the same for cyclog/multilog?)
sub check_logfile {
    my ($logfile, $old_size, $old_mtime, $old_checktime, $old_sha256_digest, $match_ref, $exclude_ref, $action_ref, $log_type, $logfile_name) = @_;
    my ($ref_logfile, $size, $mtime, $checktime, $date, $line, $idx,
	@match_hashes, @exclude_hashes, @action_hashes,
	$match, $exclude, $action, $action_value,
	@notify_arrays, @notify_lines);
    my (@session_match_flag, @session_match_array, @session_exclude_array, $session_append_string);
    my $have_matches_flag;
    my $sha256_digest = '';
    my $gzip_temp_log_flag = 0;

    if ($log_type == $LOG_TYPE_PROCESS_ACCOUNTING) {
	$ref_logfile = $PROCESS_ACCOUNTING_LOG;
    }
    elsif ($log_type == $LOG_TYPE_LINUX_JOURNAL) {
	$ref_logfile = $logfile_name;
    }
    elsif (is_gzip ($logfile)) {
	$gzip_temp_log_flag = 1;
	$ref_logfile = $logfile;
	$logfile = gunzip_logfile ($ref_logfile, $temp_dir);
    }
    else {
	$ref_logfile = $logfile;
    }

    # Note: this is sometimes operating on a temp file in the case of gzipped
    # rotated logfiles or process accounting logs. So mtime will always
    # be > old_mtime for those
    # files -- but we've already verified that that's the case before
    # we got here so it shouldn't break anything.
    # If an old rotated log is larger than the last log file we looked
    # at, we're still checking the first log line to see if we need
    # to look at the whole log file, but we only compute that when needed.
    ($size, $mtime) = get_size ($logfile);

    # Determine if we are processing a file we have already seen before,
    # or if it's a new one.
    # Not relevant to process accounting logs.
    if ($log_type == $LOG_TYPE_STANDARD_LOG ||
	$log_type == $LOG_TYPE_CYCLOG_OR_MULTILOG) {
	# If we don't have a first line SHA256 digest for the logfile,
	# get one. (Since it always gets passed in for log rotation and
	# cyclog/multilog processing, it should be non-null unless it's
	# a brand new log.)
	if ($old_sha256_digest eq '') {
	    $sha256_digest = first_log_line_sha256_digest ($logfile);
	}
	# If it's an existing logfile -- we've received as input a
	# size (old_size > 0), an mtime (old_mtime > 0) and a first
	# line SHA256 digest (old_sha256_digest ne ''), then test to
	# see if we need to reset old_size to 0 and start at the
	# beginning.
	# If file has changed (mtime > old_mtime) and the size is
	# smaller than it was OR the first line SHA256 digest doesn't
	# match, then we start over (set old_size = 0).
	if ((($old_size > 0) &&
	     ($old_mtime > 0) &&
	     ($old_sha256_digest ne '') &&
	     ($mtime > $old_mtime)) &&
	    (($size < $old_size) ||
	     (($sha256_digest = first_log_line_sha256_digest ($logfile)) ne $old_sha256_digest))) {
	    $old_size = 0;
	}
    }
    else { # process accounting logs, linux journal logs
	# We're always going to check for process accounting logs and
	# cyclogs/multilogs.  Used to say $size = $old_size + 1, but
	# since a change above, $size is size of /var/account/acct,
	# and $old_size = 0. [I don't understand this comment anymore.
	# I've just changed it so cyclogs get the test above.]
	# Process accounting logs always get old_size=0, size=1.
	$size = $old_size + 1;
    }

    # Turn match/exclude/action refs into arrays.
    @match_hashes = @{$match_ref};
    @exclude_hashes = @{$exclude_ref};
    # action_ref is an array ref of array refs - use directly.
    my $action_hashes_ref = $action_ref;

    # Identify session rules.
    # Session rules collect matches as normal with the normal processing but most will be discarded.
    # The session_exclude_array is used to find the subset of matches that are kept (so it's somewhat misnamed), and by finding
    # strings which are appended to the session_match_array used for that final filtering.
    # exclude: session-without really means: find positive matches for the "excludes," grab the capture groups from those,
    # and then use a disjunction of those to search against the results from the match: session-with regexp and report
    # all that remains. So it's really an AND of two positive matches, what's in the match and the disjunction of subparts
    # from the matching excludes.
    for ($idx = 0; $idx <= $#match_hashes; $idx++ ) {
	$session_match_flag[$idx] = 0;
	if ($match_hashes[$idx]{type} eq 'session_with') {
	    $session_match_array[$idx] = ''; # start empty
	    $session_match_flag[$idx] = 1;
	    if ($exclude_hashes[$idx]{type} eq 'session_without') {
		$session_exclude_array[$idx] = $exclude_hashes[$idx];
		# used to change exclude_hashes to 'none' or type to 'none' which impacts the next pass with rotated logs.
		# now just special-case handling of 'session_without' in match_line depending on whether returned capture
		# groups are requested.
	    }
	    else {
		# This shouldn't happen. (It would happen when type on $exclude_hashes[$idx] was changed, when processing rotated log files.)
		send_error ($ref_logfile, "Internal error - found session match hash without corresponding session exclude hash for logfile $ref_logfile/$logfile. $match_hashes[$idx]{text} / $exclude_hashes[$idx]{text}");
	    }
	}
	elsif ($exclude_hashes[$idx]{type} eq 'session_without') {
	    # This also shouldn't happen.
	    send_error ($ref_logfile, "Internal error - found session exclude hash without corresponding session match hash for logfile $ref_logfile/$logfile. $match_hashes[$idx]{text} / $exclude_hashes[$idx]{text}");
	}
    }

    # Logfile has grown.
    if ($size > $old_size) {
	if (my $log_fh = nonpriv_open_fh ($logfile, $log_type == $LOG_TYPE_CYCLOG_OR_MULTILOG)) {
	    seek ($log_fh, $old_size, 0) if ($old_size > 0);
	    while (<$log_fh>) {
		chomp;
		# Cycle through match/exclude hashes for each line.
		# Action is performed at end if notify--could be
		#   performed within this loop for other actions
		#   that might be performed on a line at a time.
		for ($idx = 0; $idx <= $#match_hashes; $idx++) {
		    $match = $match_hashes[$idx];
		    $exclude = $exclude_hashes[$idx];
#		    $action = $action_hashes[$idx]; # May not be necessary.

		    # Process session matches. If we find a match to the exclude array, the relevant match group is returned so that
		    # it can be added to the $session_match_array which will be used to identify the matches to keep.
		    # The name $session_exclude_array is misleading here, it's looking for positive matches and returning
		    # capture groups for later use.
		    if ($session_match_flag[$idx]) {
			if ($session_append_string = match_line ($session_exclude_array[$idx], $_, 1)) {
			    if ($session_match_array[$idx] eq '') {
				$session_match_array[$idx] = $session_append_string;
			    }
			    else {
				$session_match_array[$idx] .= '|' . $session_append_string;
			    }
			}
		    }

		    if (match_line ($match, $_) && !match_line ($exclude, $_)) {
			print "DEBUG: match $match->{re} and not match $exclude->{re}\nLINE: $_" if ($debug_mode && $match->{type} eq 're' && $exclude->{type} eq 're');
			if ($log_type == $LOG_TYPE_CYCLOG_OR_MULTILOG) { # cyclog or multilog
			    ($date, $line) = split (/\s+/, $_, 2);
			    if ((substr ($date, 0, 1) eq '@') && (length ($date) == 15)) { # cyclog
				$date = localtime ($date);
				push (@{$notify_arrays[$idx]}, $date . ' ' . $line);
			    }
			    elsif ((substr ($date, 0, 1) eq '@') && (length ($date) == 25) && $date =~ /^\@[\da-f]{24}$/) { # multilog
				if ($TimeTAI64_module) {
# Uncomment if using multilog/cyclog.				    
#				    $date = tai64nlocal ($date);
				}
				$line =~ s/\b([a-f0-9]{8})\b/join(".", unpack("C*", pack("H8", $1)))/eg;
				push (@{$notify_arrays[$idx]}, $date . ' ' . $line);
			    }
			    else { # unknown, leave it alone
				push (@{$notify_arrays[$idx]}, $_);
			    }
			}
			else { # syslog
			    push (@{$notify_arrays[$idx]}, $_);
			}
		    }
		}
	    }
	    close ($log_fh);

	    # If we were processing a gunzipped temp file, delete it.
	    unlink ($logfile) if ($gzip_temp_log_flag && $logfile =~ /^$temp_dir/);

	    # Perform action on corresponding matches, and reset
	    # @notify_lines for the next match/exclude/action.
	    for ($idx = 0; $idx <= $#{$action_hashes_ref}; $idx++) {
		$have_matches_flag = 0;
		my $action_arrayref = $action_hashes_ref->[$idx];

		if (exists ($notify_arrays[$idx])) {
		    # Process session matches. Just return the notify_array lines which match the collected session match strings (from
		    # the "$session_exclude_array").
		    if ($session_match_flag[$idx]) {
			# Don't grep for a null string and match everything.
			if ($session_match_array[$idx] ne '') {
			    $have_matches_flag = 1;
			    @notify_lines = grep (/$session_match_array[$idx]/, @{$notify_arrays[$idx]});
			}
		    }
		    else { # regular matches
			$have_matches_flag = 1;
			@notify_lines = @{$notify_arrays[$idx]};
		    }
		}
		if ($have_matches_flag) {
		    # Here is where to do post-processing on @notify_lines to
		    # do substitution/appending of macro names on values.
		    if ($have_postproc_macros || $have_global_postproc_macros) {
			@notify_lines = postproc_macro_substitution (@notify_lines);
		    }

		    # Execute all actions for this match/exclude pair.
		    # Actions are stored as array ref (may contain one or two actions).
		    foreach my $action_string (@{$action_arrayref}) {
			my ($action, $action_value) = split (/,/, $action_string, 2);
			
			if ($action eq 'notify') {
			    send_notify ($SHORT_HOSTNAME, $ref_logfile, $action_value, @notify_lines);
			}
			elsif ($action eq 'alert') {
			    print "***$SHORT_HOSTNAME $ref_logfile\n";
			    foreach $line (@notify_lines) {
				print "$line\n";
			    }
			    print "***\n";
			}
			elsif ($action eq 'text') {
			    # Might want to do separate text per line?
			    send_text ($SHORT_HOSTNAME, $ref_logfile, $action_value, @notify_lines);
			}
			elsif ($action eq 'execute') {
			    execute_script ($action_value, $HOSTNAME, $ref_logfile, @notify_lines);
			}
		    }
		}
	    }

	    $checktime = gettimeofday();
	    $sha256_digest = $old_sha256_digest if ($sha256_digest eq '');
	    return ($size, $mtime, $checktime, $sha256_digest);
	}
	else {
	    send_error ($ref_logfile, "Cannot open logfile $logfile. $!");
	}
    }

    $checktime = gettimeofday();
    $sha256_digest = $old_sha256_digest if ($sha256_digest eq '');
    return ($size, $mtime, $checktime, $sha256_digest);
}

# Is logfile a gzip file? Name and, if possible, magic number test.
sub is_gzip {
    my ($logfile) = @_;
    my ($gzip_fh, $magic_bytes, $magic_number);
    my $cyclog_flag = 0;
    my $raw_mode_flag = 1;

    if (substr ($logfile, length ($logfile) -3, 3) eq '.gz') {
	return 1 if (!-r $logfile); # assume it is (may require priv read)
	if ($gzip_fh = nonpriv_open_fh ($logfile, $cyclog_flag, $raw_mode_flag)) {
	    my $n = read ($gzip_fh, $magic_bytes, 2);
	    close ($gzip_fh);
	    return 0 if (!defined ($n) || $n < 2); # not a gzip
	    $magic_number = unpack ('S>', $magic_bytes); # unsigned, short, big endian
	    return $magic_number == 0x1f8b;
	}
	return 1; # if we can't open, assume it is (may require priv read)
    }

    return 0;
}

# Subroutine to unzip zipped logfile into a temp dir.
# May require privileges.
sub gunzip_logfile {
    my ($logfile, $temp_dir) = @_;
    my ($temp_in_file, $temp_out_file, $temp_logfile);

    if (!-r $logfile &&
	$privsep_flag && $nonpriv_flag) {
	$temp_logfile = get_priv_data ('gunzip', { logfile => $logfile });
	return ($temp_logfile);
    }

    $temp_in_file = basename ($logfile);
    $temp_out_file = substr ($temp_in_file, 0, length ($temp_in_file) - 3);
    $temp_logfile = $temp_dir . '/' . $temp_out_file;

    copy ($logfile, "$temp_dir/$temp_in_file");
    gunzip "$temp_dir/$temp_in_file" => $temp_logfile or
	send_error ($logfile, "gunzip failed: $GunzipError");

    # Remove in file.
    unlink ("$temp_dir/$temp_in_file");

    return ($temp_logfile);
}

# Subroutine to return a SHA256 hash of the first log line (or 0).
# This subroutine may be touching files only accessible by root,
# and is_gzip will work on such files.
# First line can be passed in if already available.
sub first_log_line_sha256_digest {
    my ($logfile, $first_line) = @_;
    my ($sha256_digest, $gzip_fh);

    # Process if $first_line is supplied and return.
    if (defined ($first_line)) {
	chomp ($first_line);
	$sha256_digest = sha256_hex ($first_line);
	return ($sha256_digest);
    }

    # Otherwise, go get it: priv case, gzip case, unpriv case.
    if (!-r $logfile && $privsep_flag && $nonpriv_flag) {
	$sha256_digest = get_priv_data ('get-hash', { logfile => $logfile });
	return ($sha256_digest);
    }
    elsif (is_gzip ($logfile)) {
	$gzip_fh = IO::Uncompress::Gunzip->new ($logfile)
	    or die "gunzip failed: $GunzipError\n";
	$first_line = <$gzip_fh>;
	close ($gzip_fh);
	if (defined ($first_line)) {
	    chomp ($first_line);
	    $sha256_digest = sha256_hex ($first_line);
	    return ($sha256_digest);
	}
    }
    elsif (open (LOG, '<', $logfile)) {
	$first_line = <LOG>;
	close (LOG);
	if (defined ($first_line)) {
	    chomp ($first_line);
	    $sha256_digest = sha256_hex ($first_line);
	    return ($sha256_digest);
	}
    }
    return 0;
}

# THIS IS NO LONGER USED, WILL KEEP AROUND FOR A WHILE IN CASE I
# DECIDE I NEED IT AGAIN. THE WEB_DATE_STRING PARSING IS INCOMPLETE.
# Replaced with SHA256 digest on first line to avoid dealing with
# parsing or timezone issues.
# Subroutine to return the parsed time of the first log line (or 0).
# DATE_STRING = standard syslog format
# DATE_STRING2 = process accounting log format
# RFC3339_date_string = RFC3339, newsyslog log rotation format
# requires setting $timezone manually below
sub first_log_line_time {
    my ($logfile) = @_;
    my ($first_line, $log_time, $date);
    my $DATE_STRING = '\w{3}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}';
    my $DATE_STRING2 = '\w{3}\s\w{3}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}';
    my $WEB_DATE_STRING = '\[\w{3}\s\w{3}\s\d{1,2}\s\d{2}:\d{2}:\d{2}\.\d{6}\s\d{4}\]';
    my $RFC3339_date_string = '\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z';
    my $timezone = 'MST';
    my $rfc3339_timezone = 'UTC';

    if (open (LOG, '<', $logfile)) {
	$first_line = <LOG>;
	close (LOG);
	if (defined ($first_line)) {
	    chomp ($first_line);
	    if ($first_line =~ /^($DATE_STRING|$DATE_STRING2|$RFC3339_date_string)/) {
		$date = $1;
		$timezone = $rfc3339_timezone if ($date =~ /^$RFC3339_date_string/);
		$log_time = parsedate ($date, PREFER_PAST => 1, ZONE => $timezone);
		return ($log_time);
	    }
	}
    }
    return 0;
}

# Identify any rotated logs which have modification times more recent than
# our last check. Was going to check here if the first one found had a line
# newer than our last check, but I don't want to unzip files twice.
# Was ignoring gzips for a few versions.
# Assumption: /var/log and all other log file locations are world-readable,
sub identify_rotated_logs {
    my ($logfile, $old_checktime) = @_;
    my ($logdir, $logbase, @files, $file, $size, $mtime);
    my ($rotated_logs_flag, @rotated_logs);
    my $first_log_time;

    $rotated_logs_flag = 0;

    # Get directory and filename of logfile from full path.
    $logdir = dirname ($logfile);
    $logbase = basename ($logfile);

    # Read directory, searching for matching files.
    # Assumes a single-digit number of rotated logs.
    # Added match for gzips; sizes may be less meaningful.
    if (opendir (DIR, $logdir)) {
	@files = grep (/^$logbase\.\d(\.gz){0,1}$/, readdir (DIR));

	# Find files that have been modified since our last check.
	foreach $file (reverse (sort (@files))) {
	    ($size, $mtime) = get_size ("$logdir/$file");
	    if ($mtime > $old_checktime) { # a rotated log that needs checking
		if (!$rotated_logs_flag) { # first one we find
		    $rotated_logs_flag = 1;
		}
		push (@rotated_logs, "$logdir/$file");
	    }
	}
	
	closedir (DIR); 
    }

    return ($rotated_logs_flag, @rotated_logs);
}

# Subroutine to read process accounting log files directly.
# Does not require root privileges on OpenBSD. Supports OpenBSD, Linux, and macOS.
# Linux support works with privsep, so the priv process doesn't need to run
# lastcomm. On Linux will call to priv process to open the first file, and
# to make readable copies of the gzipped rotated ones.
sub direct_read_process_acct_log {
    my ($acct_log, $old_size, $checktime, $old_first_line_hash, $temp_dir) = @_;
    my ($found_first_record,
	$acct_file, $acct_file_size, $acct_file_mtime,
	$processing_records);
    my ($command, $utime, $stime, $etime, $io, $btime, $uid, $gid,
	$mem, $tty, $pid, $flags);
    my ($date_time, $user, $cpu, $time, $delta, $duration);
    my ($new_size, $new_mtime, $new_checktime, $new_first_line_hash);
    my ($temp_fh, $acct_fh);
    my ($rotated_logs_flag, @rotated_acct_logs, $acct_temp_logfile);
    my ($is_live_file);

    my $OPENBSD_RECORD_FORMAT = "Z24 S< S< S< S< q< L< L< L< l< L< b32";
    my $LINUX_RECORD_FORMAT = "b8 C S< L< L< L< L< L< L< f< S< S< S< S< S< S< S< S< Z16";
    my $MACOS_RECORD_FORMAT = "Z10 S< S< S< l< L< L< S< S< l< b8";

    my $RECORD_SIZE = 64; # 64-byte records on OpenBSD and Linux.
    $RECORD_SIZE = 40 if ($^O eq 'darwin'); # 40-byte records on macOS.

    my $BLOCK_SIZE = $RECORD_SIZE * 64; # read up to 64 records at a time

    # Returns @rotated_acct_logs with changes since $checktime, sorted
    # oldest to newest, which is what we want.
    ($rotated_logs_flag, @rotated_acct_logs) = identify_rotated_logs ($acct_log, $checktime);
    # Unlike using lastcomm, which presents the records from newest to
    # oldest, we're going to start with the oldest file and read its
    # records ourselves from oldest to newest.
    push (@rotated_acct_logs, $acct_log);

    # Reset old size to zero if we don't have a checktime.
    $old_size = 0 if (!$checktime);

    $found_first_record = 0;
    $processing_records = 0;
    $is_live_file = 0;

    # Start looking at accounting files, and allow for .gz rotated
    # logs. (Don't sort here because identify_rotated_logs already
    # did it the way we want, oldest first.)
    foreach $acct_file (@rotated_acct_logs) {
	# If we're on the main file, set new size/mtime/checktime and only read to
	# that point.
	if ($acct_file eq $acct_log) {
	    $is_live_file = 1;
	    ($new_size, $new_mtime) = get_size ($acct_log);
	    $new_checktime = gettimeofday();
	}

	if (is_gzip ($acct_file)) {
	    # gunzip into a temp file and look at that (doesn't currently
	    # happen on OpenBSD).
	    $acct_temp_logfile = gunzip_logfile ($acct_file, $temp_dir);
	    $acct_file = $acct_temp_logfile;
	}

	# Open file and look for first record if we need it. Once
	# we have a first record, open temp file and start writing out
	# to it.
	if ($acct_fh = nonpriv_open_fh ($acct_file, 0, 1)) { # not cyclog, use raw mode
	    # If this is the live file, get size/mtime using fstat on the open FD
	    # instead of stat on the path (which requires privileges).
	    if ($is_live_file) {
		my @stat = stat ($acct_fh);
		if (@stat) {
		    $new_size = $stat[7]; # size
		    $new_mtime = $stat[8]; # mtime;
		}
		else {
		    die "fstat failed on live file $acct_file: $!\n";
		}
	    }
	    
	    $/ = \$RECORD_SIZE;
	    # Starting with the oldest file with an mtime > checktime.
	    # Skip past any records with time < checktime.
	    if (!$found_first_record) {
		# don't other trying to skip ahead if $old_size isn't a multiple of record size.
		if ($old_size > 0 && $old_size % $RECORD_SIZE == 0) {
		    # Read the first record and compare hash if we have an
		    # old hash. If it's "null," we use the old behavior.
		    if (defined ($old_first_line_hash)) {
			my $n_bytes = sysread ($acct_fh, my $record, $RECORD_SIZE);
			if ($n_bytes == $RECORD_SIZE &&
			    $old_first_line_hash ne 'null') {
			    $new_first_line_hash =
				first_log_line_sha256_digest ($acct_log,
							       $record);
			    seek ($acct_fh, $old_size, 0) if ($old_first_line_hash eq $new_first_line_hash);
			    # if not the live file we'll need to get a new one
			    $new_first_line_hash = undef if (!$is_live_file);
			}
			# if old hash eq 'null', use old behavior and go to
			# next-to-last record, this should only happen once
			# when switching from reportnew-1.27b or earlier.
			# Later can delete the 'null' references.
			elsif ($n_bytes == $RECORD_SIZE) {
			    seek ($acct_fh, $old_size - $RECORD_SIZE, 0);
			}
			else { # didn't get full record
			    seek ($acct_fh, 0, 0); # go back to beginning
			} # didn't get full record
		    } # have an old first-line hash
		} # $old_size > 0
	    }

	    my $position = tell ($acct_fh);

	    while (1) {
		last if ($is_live_file && $position >= $new_size); # end if we've processed everything we had at start

		# How much to read?
		my $want = $BLOCK_SIZE;
		if ($is_live_file && $position + $want > $new_size) { # limit to size at start
		    $want = $new_size - $position;
		    # Always stay record-aligned for the live file.
		    $want -= $want % $RECORD_SIZE; # end at record boundary
		    last if $want <= 0;
		}
		
		my $n_bytes = sysread ($acct_fh, my $block, $want);
		last unless defined ($n_bytes) && $n_bytes > 0;
		# Could do this at the end inside the while (1) loop
		$position += $n_bytes;

		# Process read records in block.
		for (my $offset = 0; $offset + $RECORD_SIZE <= $n_bytes; $offset += $RECORD_SIZE) {
		    my $record = substr ($block, $offset, $RECORD_SIZE);

		    # Save SHA256 hash if this is the first record of the
		    # live file and we didn't already get it above.
		    # ($position has already been incremented from 0)
		    if ($is_live_file &&
			$offset == 0 &&
			$position == $n_bytes &&
			!defined ($new_first_line_hash)) {
			$new_first_line_hash = first_log_line_sha256_digest ($acct_log, $record);
		    }
		
		    if ($^O eq 'openbsd') {
			($command, $utime, $stime, $etime,
			 $io, $btime, $uid, $gid, $mem, $tty,
			 $pid, $flags) = unpack ($OPENBSD_RECORD_FORMAT, $record);
		    }
		    elsif ($^O eq 'linux') {
			my ($version, $exitcode, $ppid, $rw, $minflt, $majflt, $swaps); # unused
			($flags, $version, $tty, $exitcode, $uid, $gid, $pid,
			 $ppid, $btime, $etime, $utime, $stime, $mem, $io,
			 $rw, $minflt, $majflt, $swaps, $command) = unpack ($LINUX_RECORD_FORMAT, $record);
		    }
		    elsif ($^O eq 'darwin') {
			($command, $utime, $stime, $etime, $btime,
			 $uid, $gid, $mem, $io, $tty, $flags) = unpack ($MACOS_RECORD_FORMAT, $record);
			$pid = 0;
		    }
		    if (!$processing_records &&
			($^O eq 'linux' ? ($btime > $checktime) :
			 ($btime > $checktime ||
			  $btime + (expand ($etime) / $AHZ) > $checktime))) {
			$found_first_record = 1;
			$processing_records = 1;

			# Create temp file (OpenBSD only).
			if ($^O eq 'openbsd') {
			    ($temp_fh, $temp_logfile) = mkstemp ("$temp_dir/reportnew.XXXXXXX");
			    if (!defined ($temp_logfile)) {
				send_error ($temp_logfile, "Could not open temp file $temp_logfile for writing. $!");
				exit; # or just return?
			    }
			}
			else { # Create temp file (Linux, macOS).
			    # create and open
			    ($temp_fh, $temp_logfile) = tempfile ("$temp_dir/reportnew.XXXXXXX");
			    if (!defined ($temp_logfile)) {
				send_error ($temp_logfile, "Could not open temp file $temp_logfile for writing. $!");
				exit; # or just return?
			    }
			}
		    }
		    if ($processing_records) {
			$date_time = ctime ($btime);
			# Remove year.
			$date_time = substr ($date_time, 0, length ($date_time) - 6);
			# Go from raw device number to name.
			if (defined ($devname_cache{$tty})) {
			    $tty = $devname_cache{$tty};
			}
			elsif ($^O eq 'linux') {
			    $tty = linux_device ($tty);
			}
			else {
			    $tty = '??';
			}
			# Convert uid to user (or use uid if no user in passwd file).
			$user = getpwuid($uid) || $uid;
			# Command + pid. No pid on macOS.
			# This introduces a difference from lastcomm on Linux that must
			# be accounted for in match/exclude rules (commands aren't followed
			# by spaces). Duration is also now available without fakery.
			$command = "$command\[$pid\]" unless ($^O eq 'darwin');
			# Process flags.
			$flags = flagbits ($flags);
			# User time plus system time.
			$cpu = (expand ($utime) + expand ($stime)) / $AHZ;
			$cpu = sprintf "%.2f", $cpu; # was %6.2f for formatted cols
			$delta = expand ($etime) / $AHZ;
			$duration = sprintf "%1.0f:%02.0f:%05.2f", $delta / SECSPERHOUR,
			    fmod ($delta, SECSPERHOUR) / SECSPERMIN,
			    fmod ($delta, SECSPERMIN);
			print $temp_fh "$date_time $tty $user $command $flags $cpu secs $duration\n";
		    }
		} # for loop for records in block
	    } # while (1)
	} # open
	else {
	    send_error ($acct_file, "Could not open acct file $acct_file. $!");
	    exit;
	}
	close ($acct_fh);
	$/ = $NL;
    }

    if (!$found_first_record || !$processing_records) {
	# This should provoke an error and exit, we never wrote anything
	# out. This can happen if process accounting is turned off.
	send_error ($acct_log, "Could not find any new records in acct file $acct_log (direct method). This shouldn't happen--is process accounting on?");
	exit;
    }
    # Otherwise, close the temp file and return log path.
    close ($temp_fh);

    # if we didn't get a new one, return the old one
    $new_first_line_hash = $old_first_line_hash if (!defined ($new_first_line_hash));

    return ($temp_logfile, $new_size, $new_mtime, $new_checktime, $new_first_line_hash);
}

# Subroutine to expand accounting time.
sub expand {
    my ($time) = @_;
    my $newtime;

    $newtime = $time & 017777;
    $time >>= 13;
    while ($time) {
        $time--;
        $newtime <<= 3;
    }
    return ($newtime);
}

# Subroutine to return process flags in human-readable form.
# Might want to re-do this with unpacking flags to hex and
# using bitwise operators. (0x02 was ASU, now removed)
#    vec ($AFORK, 0, 8) = 0x01; # F
#    vec ($AMAP, 0, 8) = 0x04; # M
#    vec ($ACORE, 0, 8) = 0x08; # D
#    vec ($AXSIG, 0, 8) = 0x10; # X
#    vec ($APLEDGE, 0, 8) = 0x20; # P
#    vec ($ATRAP, 0, 8) = 0x40; # T
#    vec ($AUNVEIL, 0, 8) = 0x80; # U
#    vec ($APINSYS, 0, 8) = 0x200; # S
#    vec ($ABTCFI, 0, 8) = 0x400; # B
sub flagbits {
    my ($flag) = @_;
    my ($output, $idx);
    my @flagbitmap = (
	[ 0, 'F' ], # fork'd but not exec'd
	[ 2, 'M' ], # killed by syscall or stack mapping violation
	[ 3, 'D' ], # dumped core
	[ 4, 'X' ], # killed by a signal
	[ 5, 'P' ], # killed due to pledge violation
	[ 6, 'T' ], # memory access violation
	[ 7, 'U' ], # unveil access violation
	[ 9, 'S' ], # killed by syscall pin violation
	[ 10, 'B' ] # BT CFI violation
	);
    @flagbitmap = (
	[ 1, 'S' ], # process executed with superuser privs
	[ 0, 'F' ], # fork'd but not exec'd
	[ 2, 'C' ], # ACOMPAT, compatibility mode (VAX/PDP, obsolete)
	[ 3, 'D' ], # dumped core
	[ 4, 'X' ]  # killed by a signal
	) if ($^O eq 'linux' || $^O eq 'darwin');

    $output = '-'; # this is OpenBSD-style, Linux style would omit dash if any other flags are present
    for my $bitmap_ent (@flagbitmap) {
	my ($idx, $ch) = @$bitmap_ent;
	$output .= $ch if (substr ($flag, $idx, 1) eq '1');
    }

    return ($output);
}

# Subroutine to return device names for Linux (and cache them).
sub linux_device {
    my ($dev) = @_;
    my ($devname, $major, $minor);

    $major = ($dev >> 8) & 0xff;
    $minor = $dev & 0xff;

    if ($major == 4) {
	$devname = "tty$minor"; # virtual console
    }
    elsif ($major >= 136 && $minor <= 143) {
	my $pts = ($major - 136) * 256 + $minor;
	$devname = "pts/$pts"; # UNIX98 pts
    }
    elsif ($major == 5 && $minor == 1) {
	$devname = 'console'; # system console
    }
    elsif ($dev == 0) {
	$devname = '__';
    }
    else {
	$devname = "tty($major, $minor)"; # fallback
    }

    $devname_cache{$dev} = $devname; # cache it
    return $devname;
}

# Subroutine to read process accounting log files using lastcomm.
# Does not require root privileges on BSDs (including macOS), but
# does on Linux.
# Linux also gzips rotated process accounting logs and expects them
# to be piped to lastcomm with zcat.
sub read_process_acct_log {
    my ($acct_log, $checktime, $temp_dir) = @_;
    my ($idx, $acct_file, @lastcomm_logs, $log_line, $temp_logfile);
    my ($rotated_logs_flag, @rotated_acct_logs, $acct_temp_logfile);
    my ($command, $flags, $user, $tty, $time_info,
	$cpu, $secs, $day_name, $month, $day, $time, $duration,
	$time_dec);
    my ($new_checktime);
    my ($temp_fh);
    # last field A30+, no duration
    my $linux_format = 'A16 A7 A9 A9 A*';
    # last field A44+, includes duration
    my $macos_format = 'A11 A8 A9 A9 A*';
    my ($req_response); # for privsep

    if ($^O eq 'linux' && $privsep_flag && $nonpriv_flag) {
	$req_response = get_priv_data ('get-linux-pacct', { logfile => $acct_log, checktime => $checktime });
	if (defined ($req_response)) {
	    $temp_logfile = $req_response->{temp_logfile};
	    $new_checktime = $req_response->{checktime};
	    return ($temp_logfile, $new_checktime);
	}
	send_error ($acct_log, "Received unexpected response from privileged parent process trying to get Linux process accounting from $logfile.");
	exit; # or just return?
    }

    # Read process accounting logs, out to end or to $checktime, whichever
    # comes first.  Write out to temp file, in reverse order, with more
    # standardized date/time stamps.  Return $temp_logfile name.

    # Do lastcomm, starting with the most recent and working backward,
    # reading lines into @lastcomm_logs with parsed date/time, until
    # we reach $checktime or run out of logs.

    # This algorithm doesn't quite work, because lastcomm's times are
    # the time the process started, but the record is written when it
    # ends, so the records can be out of order.
    # This could be fixed by checking that the time of the new log
    # is earlier, even if the duration is added to it (or if the
    # time is earlier by more than the duration).

    # Only looks at acct.N[.gz] where N is a single digit, Linux will go to
    # double digits. Also returns oldest first.
    # $rotated_logs_flag ignored, we're always going to do something
    # unless there's no log at all, which should already have been
    # determined (unless it just got deleted).
    ($rotated_logs_flag, @rotated_acct_logs) = identify_rotated_logs ($acct_log, $checktime);
    # Insert current log at the front. It will be sorted anyway, but
    # this puts it where it belongs.
    unshift (@rotated_acct_logs, $acct_log);

    # Set new checktime to roughly correspond with the last log entry
    # (first one in the newest file, which we process first).
    $new_checktime = gettimeofday();
    # Sorting from oldest first to newest first. Maybe make an option
    # to identify_rotated_logs to avoid unnecessary sort.
    # Note that sort here and in identify_rotated_logs won't work
    # right if support is added for more than one digit in the filename,
    # unless we sort numerically on the number and keep the one without
    # a number at the front.
    foreach $acct_file (sort @rotated_acct_logs) {
	if (is_gzip ($acct_file)) {
	    # gunzip into a temp file and run lastcomm on that.
	    $acct_temp_logfile = gunzip_logfile ($acct_file, $temp_dir);
	    $acct_file = $acct_temp_logfile;
	}

	if (!open (LASTCOMM, '-|', $LASTCOMM, '-f', $acct_file)) {
	    send_error ($acct_file, "Could not open acct file $acct_file. $!");
	    exit; # or just return?
	}
	while (<LASTCOMM>) {
	    # Parse line, check time.  If lastcomm time <= $check_time,
	    # then exit with "last."
	    chomp;

	    # Linux has older format with no duration.
	    if ($^O eq 'linux') {
	       ($command, $flags, $user, $tty, $time_info) = unpack ($linux_format, $_);
	       $flags =~ s/^\s*//;
	       $time_info =~ s/^\s*//;
	       ($cpu, $secs, $day_name, $month, $day, $time) = split (/\s+/, $time_info, 6);
	       $duration = '(0:00:00.00)'; # fake it for the calculation
	    }
	    elsif ($^O eq 'darwin') {
		# macOS can have spaces in command names.
		($command, $flags, $user, $tty, $time_info) = unpack ($macos_format, $_);
		$flags =~ s/^\s*//;
		$time_info =~ s/^\s*//;
		($cpu, $secs, $day_name, $month, $day, $time, $duration) = split (/\s+/, $time_info, 7);
	    }
	    else { # OpenBSD, macOS
    	    	 ($command, $flags, $user, $tty, $time_info) = split (/\s+/, $_, 5);
	    	 ($cpu, $secs, $day_name, $month, $day, $time, $duration) = split (/\s+/, $time_info, 7);
	    }
	    $time = $day_name . ' ' . $month . ' ' . $day . ' ' . $time;
	    $time_dec = parsedate ($time, PREFER_PAST => 1);
	    $duration =~ s/^\((.*)\)$/$1/;

	    # Stop this once we reach an entry earlier than last check.
	    # Correcting for long-duration processes started a long
	    # time ago that are out of sequence.
	    if ($time_dec < $checktime) {
		my ($dur_hours, $dur_mins, $dur_secs) = split (/:/, $duration);
		if ($checktime - $time_dec > ($dur_hours * SECSPERHOUR +
					      $dur_mins * SECSPERMIN +
					      $dur_secs)) {
		    unlink ($acct_temp_logfile) if (defined ($acct_temp_logfile) &&
						    $acct_file eq $acct_temp_logfile);
		    last;
		}
	    }
	    
	    $log_line = "$time $tty $user $command $flags $cpu $secs";
	    $log_line .= " $duration" unless ($^O eq 'linux');

	    push (@lastcomm_logs, $log_line);
	}
	close (LASTCOMM);
	unlink ($acct_temp_logfile) if (defined ($acct_temp_logfile) &&
					$acct_file eq $acct_temp_logfile);
    }

    # Now write out @lastcomm_logs to a new temp file in reverse order.

    # Create temp file.
    if ($^O eq 'openbsd') {
	($temp_fh, $temp_logfile) = mkstemp ("$temp_dir/reportnew.XXXXXXX");
	if (!defined ($temp_logfile)) {
	    send_error ($temp_logfile, "Could not open temp file $temp_logfile for writing. $!");
	    exit; # or just return?
	}
    }
    else {
	# create and open
	($temp_fh, $temp_logfile) = tempfile ("$temp_dir/reportnew.XXXXXXX");
	if (!defined ($temp_logfile)) {
	    send_error ($temp_logfile, "Could not open temp file $temp_logfile for writing. $!");
	    exit; # or just return?
	}
	chomp ($temp_logfile);
    }

    # Print out each log line to the temp file.
    foreach $log_line (reverse (@lastcomm_logs)) {
	print $temp_fh "$log_line\n";
    }

    # Close temp file.
    close ($temp_fh);

    # Return the filename.
    return ($temp_logfile, $new_checktime);
}

# Read log information from Linux journal into a temp file to process.
# This subroutine trusts the config parser to have validated the
# arguments.
# Requires privileges.
sub read_linux_journal {
    my ($logfile, $checktime, $temp_dir) = @_;
    my ($temp_logfile, $temp_fh, $new_checktime);
    my @selection_args;
    my $NO_ENTRIES = '-- No entries --';
    my $result; # for privilege separation

    if ($privsep_flag && $nonpriv_flag) {
	$result = get_priv_data ('get-journal', { logfile => $logfile, checktime => $checktime });
	return ($result->{temp_logfile}, $result->{checktime});
    }

    # Create temp file.
    # create and open
    ($temp_fh, $temp_logfile) = tempfile ("$temp_dir/reportnew.XXXXXXX");
    if (!defined ($temp_logfile)) {
	send_error ($temp_logfile, "Could not open temp file $temp_logfile for writing. $!");
	exit; # or just return?
    }
    chomp ($temp_logfile);

    # Convert last checktime to English.
    my $since_param;
    if ($checktime != 0) {
	$since_param = sprintf("\@%d", $checktime);  # journalctl epoch syntax
    }

    # Formulate selection arguments.
    if ($logfile =~ /journal unit ([\w\-\.]+)$/) {
	my $unit_arg = $1;
	if (!grep ($_ eq $unit_arg, @linux_journal_units)) {
	    send_error ($logfile, "Specified journal unit $unit_arg is not enabled.");
	    exit; # or just return?
	}
	@selection_args = ('-u', $unit_arg);
    }
    elsif ($logfile =~ /journal syslog-id ([\w\-\.]+)$/) {
	my $syslogid_arg = $1;
	# no check today
	@selection_args = ('-t', $syslogid_arg);
    }
    elsif ($logfile =~ /journal syslog-facility ([\w\-\.]+)$/) {
	my $syslog_facility_arg = $1;
	if (!grep ($_ eq $syslog_facility_arg, @linux_journal_syslog_facilities)) {
	    send_error ($logfile, "Specified journal syslog facility $syslog_facility_arg is not defined.");
	    exit; # or just return?    
	}
	@selection_args = ('--facility=' . $syslog_facility_arg);
    }
    else {
	send_error ($logfile, "Could not parse journal logfile name. $logfile");
	exit; # or just return?
    }
    
    # Append date if checktime nonzero.
    push (@selection_args, '--since', $since_param) unless (!$checktime);

    # Open journalctl to read log lines.
    if (!open (JOURNAL, '-|', $JOURNALCTL, @selection_args)) {
	send_error ($logfile, "Could not open file handle for $JOURNALCTL. $!");
	exit; # or just return?
    }

    while (<JOURNAL>) {
	print $temp_fh "$_" unless (/^$NO_ENTRIES$/);
    }

    # Set new checktime.
    $new_checktime = gettimeofday();

    # Close journalctl.
    close (JOURNAL);

    # Close temp file.
    close ($temp_fh);
    
    return ($temp_logfile, $new_checktime);
}

# Return 1 if a regexp is valid, 0 if not.
sub valid_regexp {
    my ($regexp) = @_;

    if (($regexp =~ /\/.*\//) ||
	($regexp =~ /!\/.*\//)) {
	return 1;
    }
    else {
	return 0;
    }
}

# Read the contents of the size file.
# If a current_logfile is specified, do not update hashes for that logfile.
sub read_size_file {
    my ($size_file, $current_logfile) = @_;
    my ($logfile, $size, $mtime, $checktime, $sha256_digest, $processing_pid);

    if (!-e $size_file) {
	foreach $logfile (@logfiles) {
	    $log_size{$logfile} = 0;
	    $log_mtime{$logfile} = 0;
	    $log_checktime{$logfile} = 0;
	    $log_sha256_digest{$logfile} = '';
	    $log_processing_pid{$logfile} = 0;
	}
	$processing_pid = 0;
    }
    else {
	if (open (SIZEFILE, '<', $size_file)) {
	    if (!flock (SIZEFILE, LOCK_SH)) {
		send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot lock size file for reading. $!");
	    }
	    while (<SIZEFILE>) {
		chomp;
		($logfile, $size, $mtime, $checktime, $sha256_digest, $processing_pid) = split (/,/);
		if (!defined ($processing_pid)) { # bad line in size file
		    send_notify ($SHORT_HOSTNAME, $config_file, $master_notify, "Malformed line in size file. $_");
		}
		if (!grep ($_ eq $logfile, @logfiles)) {
		    send_notify ($SHORT_HOSTNAME, $config_file, $master_notify, "Logfile $logfile in size file is not in config file.");
		}
		if (!defined ($current_logfile) || $current_logfile ne $logfile) {
		    $log_size{$logfile} = $size;
		    $log_mtime{$logfile} = $mtime;
		    $log_checktime{$logfile} = $checktime;
		    if (!defined ($sha256_digest)) { # old size file
			$sha256_digest = '';
		    }
		    elsif ($sha256_digest eq 'null') { # how we store null ones
			$sha256_digest = '';
		    }
		    $log_sha256_digest{$logfile} = $sha256_digest;
		    if (!defined ($processing_pid)) { # old size file
			$processing_pid = 0;
		    }
		    $log_processing_pid{$logfile} = $processing_pid;
		}
	    }
	    if (!flock (SIZEFILE, LOCK_UN)) {
		send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot unlock size file after reading. $!");
	    }
	    close (SIZEFILE);
	}
	else {
	    send_error ($size_file, "Cannot open size file. $!");
	    exit;
	}

	if (!defined ($current_logfile)) {
	    foreach $logfile (@logfiles) {
		if (!defined ($log_size{$logfile})) {
		    send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Logfile $logfile in config file is not in size file.");
		    $log_size{$logfile} = 0;
		    $log_mtime{$logfile} = 0;
		    $log_checktime{$logfile} = 0;
		    $log_sha256_digest{$logfile} = '';
		    $log_processing_pid{$logfile} = 0;
		    write_size_file ($size_file, $logfile, $LOG_APPEND);
		}
	    }
	}
    }
}

# Get current log file size and mtime.
# Doesn't use privs, could conceivably require them for some logs.
sub get_size {
    my ($logfile) = @_;
    my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime,
	$mtime, $ctime, $blksize, $blocks);

    ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime,
     $mtime, $ctime, $blksize, $blocks) = stat $logfile;

    $size = 0 if (!defined ($size));
    $mtime = 0 if (!defined ($mtime));

    return ($size, $mtime);
}

# Subroutine to precompile regexps for matching.
sub compile_re {
    my ($pattern, $context) = @_;
    my $re = eval { qr/$pattern/ };
    die "Invalid regex in $context: $@" if $@;
    return $re;
}

# Return 1 if line matches regexp, 0 if not.
# If $return_group is specified, on match the first capture group is returned.
sub match_line {
    my ($rule, $line, $return_group) = @_;

    if (!defined ($return_group)) {
	$return_group = 0;
    }

    # rule types 'all' or 'none'
    return 1 if ($rule->{type} eq 'all');
    return 0 if ($rule->{type} eq 'none');
    # rule type 'session_without' treated same as 'none'
    # if no return_group is requested
    return 0 if ($rule->{type} eq 'session_without' && !$return_group);

    # rule types 're', 'session_with'; when 'session_without' is used
    # to collect capture groups for positive matches it comes here;
    # all are compiled regexps.
    if ($line =~ $rule->{re}) {
	# negative rule: match only if it does not match
	if ($rule->{neg}) {
	    return 0;
	}
	# positive rule, return $1 if required and present
	return $1 if (defined ($1) && $return_group);
	return 1;
    }
    # regex didn't match
    else {
	return 1 if ($rule->{neg});
	return 0;
    }
}

# Execute a script as nonprivileged user. (If non privsep, drop
# privileges and run as nobody/nogroup.) Script must be signed.
# If OpenBSD, re-pledge? Need to allow writing to the file system
# as a prime use case would be, e.g., extracting and logging IP
# addresses, mail certs, etc.
sub execute_script {
    my ($script_name, $hostname, $logfile, @lines) = @_;
    my ($script_path, $pid);

    # Construct full script path
    $script_path = "$SCRIPT_DIR/$script_name";

    # Re-validate script name (defense in depth)
    if ($script_name !~ /^[\w\-\.]+$/) {
	send_error ($script_path, "Invalid script name in execute_script: $script_name");
	return;
    }

    # TOCTOU mitigation: Re-verify script existence and signature just before execution
    if (!-r $script_path || !-f $script_path || -l $script_path) {
	send_error ($script_path, "Execute script $script_path is not readable, not a regular file, or is a symlink");
	return;
    }

    if (!verify_signify_sig ($script_path)) {
	send_error ($script_path, "Cannot verify signify signature on execute script $script_path just before execution");
	return;
    }

    # Create a pipe to pass matching lines to the script
    pipe (my $read_fh, my $write_fh) || do {
	send_error ($script_path, "Cannot create pipe for execute script. $!");
	return;
    };

    # Fork to execute script
    $pid = fork();
    if (!defined ($pid)) {
	send_error ($script_path, "Cannot fork to execute script $script_path. $!");
	close ($read_fh);
	close ($write_fh);
	return;
    }

    # Parent process
    if ($pid) {
	close ($read_fh);
	
	# Write matching lines to the pipe
	foreach my $line (@lines) {
	    print $write_fh "$line\n";
	}
	close ($write_fh);
	
	# Wait for child to complete
	waitpid ($pid, 0);
	my $exit_status = $? >> 8;
	if ($exit_status != 0) {
	    send_error ($script_path, "Execute script $script_path exited with status $exit_status");
	}
	return $exit_status;
    }

    # Child process - will exec the script
    close ($write_fh);

    # Rename process.
    $0 = 'reportnew script exec';
    
    # Drop privileges to nobody/nogroup unless using privsep or being run
    # by a non-root user.
    if ($< == 0 || $> == 0) { # Running as root (real or effective uid is 0)
	my ($nobody_uid, $nobody_gid);
    
	# Use nobody:nogroup (and nobody:nobody as fallback).
	$nobody_uid = getpwnam ('nobody');
	$nobody_gid = getgrnam ('nogroup') || getgrnam ('nobody');
    
	if (defined ($nobody_uid) && defined ($nobody_gid)) {
	    # Drop privileges
	    eval {
		require Privileges::Drop;
		Privileges::Drop::drop_uidgid ($nobody_uid, $nobody_gid);
	    };
	    if ($@) {
		# If we can't load Privileges::Drop, try manual drop
		$( = $nobody_gid;
		$) = $nobody_gid;
		$< = $nobody_uid;
		$> = $nobody_uid;
	    
		# Verify privilege drop worked
		if ($> != $nobody_uid || $< != $nobody_uid ||
		    $) != $nobody_gid || $( != $nobody_gid) {
		    die "Failed to drop privileges for execute script\n";
		}
	    }
	}
	else {
	    die "Cannot find nobody:nogroup (or nobody:nobody) for privilege drop for script execution.\n";
	}
    }

    # Redirect stdin from the pipe
    open (STDIN, '<&', $read_fh) || die "Cannot redirect stdin. $!\n";
    close ($read_fh);
    
    # Set up environment variables
    $ENV{'REPORTNEW_HOSTNAME'} = $hostname;
    $ENV{'REPORTNEW_LOGFILE'} = $logfile;
    
    # Clean environment for security
    delete @ENV{qw(IFS CDPATH BASH_ENV PERL5LIB LD_PRELOAD LD_LIBRARY_PATH)};

    # Exec the script
    # The child inherits the unveil restrictions from the parent process.
    # On OpenBSD, the script won't be able to access files outside the
    # unveiled paths (scripts directory and /tmp).
    # The child also inherits pledges - it can exec this one script, but
    # the script itself won't be able to exec other programs or fork because
    # shell scripts executed via exec don't inherit pledges, they run under
    # the kernel's default restrictions for the executable format.
    exec ($script_path) || die "Cannot exec $script_path. $!\n";
}

# Send a mail notification.
sub send_notify {
    my ($hostname, $logfile, $notify_list, @lines) = @_;
    my ($line);

    if (!defined ($notify_list)) {
	if (!defined ($master_notify)) {
	    # This seems to happen periodically, indicating a bug
	    # in the config parsing.
	    $notify_list = $SECURITY_ADMIN;
	}
	else {
	    $notify_list = $master_notify;
	}
    }

    # Defensive checks against undef parameters
    $hostname = $SHORT_HOSTNAME if (!defined ($hostname));
    $logfile = '(unknown)' if (!defined ($logfile));
    $email_sender = $EMAIL_SENDER if (!defined ($email_sender));

    if ($debug_mode) {
	print "$hostname: $logfile ($notify_list)\n";
	foreach $line (@lines) {
	    print "$line\n";
	}
    }

    else {
	if (open (MAIL, '|-', $SENDMAIL, '-t')) {
	    print MAIL "From: Reporter <$email_sender>\n";
	    print MAIL "To: $notify_list\n";
	    print MAIL "Subject: $hostname $logfile\n\n";
	    s/\r//g for @lines; # remove carriage returns
	    foreach $line (@lines) {
		if (length($line) > 990) {
		    print MAIL "$_\n" for ($line =~ m/.{1,990}/gs);
		}
		else {
		    print MAIL "$line\n";
		}
	    }
	    close (MAIL);
	}
	else {
	    die "Unable to open $SENDMAIL. $!\n";
	}
    }
}

# Send a text message. This code could easily be merged with
# send_notify, it is identical except for the mail format.
sub send_text {
    my ($hostname, $logfile, $notify_list, @lines) = @_;
    my ($line);

    if (!defined ($notify_list)) {
	if (!defined ($master_notify)) {
	    # This seems to happen periodically, indicating a bug
	    # in the config parsing.
	    $notify_list = $SECURITY_ADMIN;
	}
	else {
	    $notify_list = $master_notify;
	}
    }

    # Defensive checks against undef parameters
    $hostname = $SHORT_HOSTNAME if (!defined ($hostname));
    $logfile = '(unknown)' if (!defined ($logfile));
    $email_sender = $EMAIL_SENDER if (!defined ($email_sender));

    if ($debug_mode) {
	print "$hostname: $logfile ($notify_list)\n";
	foreach $line (@lines) {
	    print "$line\n";
	}
    }

    else {
	if (open (MAIL, '|-', $SENDMAIL, '-t')) {
	    print MAIL "From: Reporter <$email_sender>\n";	    
	    print MAIL "To: $notify_list\n\n";
	    s/\r//g for @lines; # remove carriage returns
	    foreach $line (@lines) {
		if (length($line) > 990) {
		    print MAIL "$_\n" for ($line =~ m/.{1,990}/gs);
		}
		else {
		    print MAIL "$line\n";
		}
	    }
	    close (MAIL);
	}
	else {
	    die "Unable to open $SENDMAIL. $!\n";
	}
    }
}

# Send error notification.
sub send_error {
    my ($filename, $error) = @_;

    if (defined ($master_notify)) {
	send_notify ($SHORT_HOSTNAME, $filename, $master_notify, $error);
    }
    else {
	send_notify ($SHORT_HOSTNAME, $filename, $SECURITY_ADMIN, $error);
    }
}

# Write out size file.
# This is always called immediately after a read_size_file, though one
# which might skip the current logfile.
# If current_logfile is not specified or the size file doesn't exist
# or can't be opened for reading but can be opened for writing and locked,
# we write out everything.
# If it's a new logfile that isn't in the size file, we just append it
# to the end.
# Otherwise, we read everything in and overwrite with the just-read data
# except for the log file we're updating.
sub write_size_file {
    my ($size_file, $current_logfile, $start_or_end) = @_;
    my ($logfile, $sha256_digest, $my_pid);
    my ($write_all_logfiles,
	$lock_timeout_limit, @size_file_lines, $size_file_line);

    # If we're just adding a new logfile to the end of the size file,
    # do that.
    if (defined ($current_logfile) && $start_or_end == $LOG_APPEND) {
	if (!open (SIZEFILE, '>>', $size_file)) {
	    send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot open size file for appending. $!");
	    exit;
	}
	$lock_timeout_limit = time() + $SIZE_FILE_LOCK_TIMEOUT_LIMIT;
	until (flock (SIZEFILE, LOCK_EX | LOCK_NB)) {
	    if (time() > $lock_timeout_limit) {
		send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot lock size file for writing. $!");
		exit;
	    }
	    sleep 1;
	}
	print SIZEFILE "$current_logfile,$log_size{$current_logfile},$log_mtime{$current_logfile},$log_checktime{$current_logfile},null,0\n";
	if (!flock (SIZEFILE, LOCK_UN)) {
	    send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot unlock size file after appending. $!");
	    exit;
	}
	close (SIZEFILE);
	return;
    }

    # If we're just changing a single logfile, read in the current
    # size file without parsing (but we do lock).
    if (defined ($current_logfile) && (-e $size_file)) {
	$write_all_logfiles = 0;
	if (open (SIZEFILE, '<', $size_file)) {
	    if (flock (SIZEFILE, LOCK_SH)) {
		@size_file_lines = <SIZEFILE>;
	    }
	    else {
		send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot lock size file for reading. $!");
		exit;
	    }
	}
	else {
	    send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot open size file for reading. $!");
	    exit;
	}
	if (!flock (SIZEFILE, LOCK_UN)) {
	    send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot unlock size file after reading. $!");
	    exit;
	}
	close (SIZEFILE);
    }
    else {
	$write_all_logfiles = 1;
    }

    # Now, we write everything out, either with what we have or with what
    # was just read from the size file except for our current logfile.
    # Modified to remove race condition (was truncating before lock).
    if (!sysopen (SIZEFILE, $size_file, O_RDWR | O_CREAT, 0600)) {
	send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot open size file for writing. $!");
	exit;
    }

    $lock_timeout_limit = time() + $SIZE_FILE_LOCK_TIMEOUT_LIMIT;
    until (flock (SIZEFILE, LOCK_EX | LOCK_NB)) {
	if (time() > $lock_timeout_limit) {
	    send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot lock size file for writing. $!");
	    exit;
	}
	sleep 1;
    }

    # Now have lock, truncate.
    truncate (SIZEFILE, 0) || do {
	send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot truncate size file. $!");
	exit;
    };

    # If we're writing out all log files, do that.
    if ($write_all_logfiles) {
	foreach $logfile (@logfiles) {
	    # Write out 'null' if blank. Should only be for process accounting logs.
	    if (!defined ($log_sha256_digest{$logfile})) {
		$sha256_digest = 'null';
	    }
	    else {
		$sha256_digest = $log_sha256_digest{$logfile};
		$sha256_digest = 'null' if ($sha256_digest eq '');
	    }

	    # Write it all out.
	    if ($logfile eq $current_logfile && $start_or_end == $LOG_PROCESSING_START) {
		$my_pid = $$;
	    }
	    else {
		$my_pid = 0;
	    }
	    print SIZEFILE "$logfile,$log_size{$logfile},$log_mtime{$logfile},$log_checktime{$logfile},$sha256_digest,$my_pid\n";
	}
    }
    else { # just changing current log info
	foreach $size_file_line (@size_file_lines) {
	    chomp ($size_file_line);
	    ($logfile) = split (/,/, $size_file_line);
	    if ($logfile eq $current_logfile) {
		# Write out 'null' if blank. Should only be for process accounting logs.
		if (!defined ($log_sha256_digest{$logfile})) {
		    $sha256_digest = 'null';
		}
		else {
		    $sha256_digest = $log_sha256_digest{$logfile};
		    $sha256_digest = 'null' if ($sha256_digest eq '');
		}

		# Write it all out.
		if ($logfile eq $current_logfile && $start_or_end == $LOG_PROCESSING_START) {
		    $my_pid = $$;
		}
		else {
		    $my_pid = 0;
		}
		print SIZEFILE "$logfile,$log_size{$logfile},$log_mtime{$logfile},$log_checktime{$logfile},$sha256_digest,$my_pid\n";
	    }
	    else {
		print SIZEFILE "$size_file_line\n";
	    }
	}
    }

    # Unlock and close size file.
    if (!flock (SIZEFILE, LOCK_UN)) {
	send_notify ($SHORT_HOSTNAME, $size_file, $master_notify, "Cannot unlock size file after writing. $!");
	exit;
    }
    close (SIZEFILE);
}

# Subroutine to return 1 if process exists, 0 otherwise.
sub pid_exists {
    my ($pid) = @_;

    if (kill (0, $pid) || $!) {
	return 1 unless $! eq 'No such process';
    }

    return 0;
}
    
# Subroutine to replace macro names with values on preprocessing.
# Process host macros then global macros.
sub preproc_macro_substitution {
    my ($orig_regexp_line, $line_num) = @_;
    my ($more_to_process, $regexp_line, $macro_name, $global_macro);

    $regexp_line = $orig_regexp_line;

    $more_to_process = 1;

    while ($more_to_process) {
	if ($regexp_line =~ /%%([\w\-]+)%%/) {
	    $macro_name = $1;
	    $global_macro = 0;
	    if (!defined ($preproc_macro{$macro_name})) {
		if (defined ($global_preproc_macro{$macro_name})) {
		    $global_macro = 1;
		}
		elsif ($config_check) {
		    # Non-fatal error in config check mode. Leave macro unexpanded.
		    print "Warning: Undefined preproc macro %%$macro_name%% on line $line_num. $orig_regexp_line\n";
		    return ($regexp_line); # End processing on this line for config check.
		}
		else {
		    send_error ($config_file, "Undefined preproc macro %%$macro_name%% on line $line_num. $orig_regexp_line");
		    exit;
		}
	    }
	    # Replace all occurrences.
	    if ($global_macro) {
		$regexp_line =~ s/%%$macro_name%%/$global_preproc_macro{$macro_name}/g;	
	    }
	    else {
		$regexp_line =~ s/%%$macro_name%%/$preproc_macro{$macro_name}/g;
	    }
	}
	else {
	    $more_to_process = 0;
	}
    }
    return ($regexp_line);
}

# Subroutine to rewrite output containing macro values by
# appending or substituting macro names. This would work
# better if only executed where we know the macro names were
# present in preprocessing.
# We have to go through all the macros on each line.
# Bad stuff could happen if there are macro values that match
# macro names.
# Process host macros first, then global macros.
sub postproc_macro_substitution {
    my (@output_lines) = @_;
    my ($output_line, $macro_value, $macro_name);

    my $APPEND = 1;
    my $SUBSTITUTE = 0;

    if ($have_postproc_macros) {
	foreach $output_line (@output_lines) {
	    foreach $macro_value (keys (%append_macro)) {
		if ($output_line =~ /$macro_value/) {
		    $output_line = postproc_match_and_replace ($output_line, $macro_value, $append_macro{$macro_value}, $APPEND);
		}
	    }
	    foreach $macro_value (keys (%substitute_macro)) {
		if ($output_line =~ /$macro_value/) {
		    $output_line = postproc_match_and_replace ($output_line, $macro_value, $substitute_macro{$macro_value}, $SUBSTITUTE);	    
		}
	    }
	}
    }

    if ($have_global_postproc_macros) {
	foreach $output_line (@output_lines) {
	    foreach $macro_value (keys (%global_append_macro)) {
		if ($output_line =~ /$macro_value/) {
		    $output_line = postproc_match_and_replace ($output_line, $macro_value, $global_append_macro{$macro_value}, $APPEND);
		}
	    }
	    foreach $macro_value (keys (%global_substitute_macro)) {
		if ($output_line =~ /$macro_value/) {
		    $output_line = postproc_match_and_replace ($output_line, $macro_value, $global_substitute_macro{$macro_value}, $SUBSTITUTE);	    
		}
	    }
	}
    }
    
    return (@output_lines);
}

# Subroutine to do post-processing macro matching and replacement,
# with special handling for IP addresses.
sub postproc_match_and_replace {
    my ($line, $macro_value, $macro_name, $append) = @_;

    # Special case handling of macro value matches for IP addresses.
    # Only match and replace if the IP address is anchored on the left
    # side by beginning of line, colon, or left bracket and on the right side
    # by end of line, colon, period, or right bracket.
    #
    # Note that this special casing is NOT done on the front end (preproc
    # macros), as we assume the match rule will contain the context for
    # anchoring the match--if we stuck things in it could break a rule
    # and would be very counter-intuitive to the builder of the config.
    my $IPv4_REGEXP = '^\d{1,3}\\\.\d{1,3}\\\.\d{1,3}\\\.\d{1,3}$';
    my $IPv6_REGEXP = '^(((?=(?>.*?::)(?!.*::)))(::)?(([0-9A-F]{1,4})::?){0,5}|((?5):){6})(\2((?5)(::?|$)){0,2}|((25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.|$)){4}|(?5):(?5))(?<![^:]:)(?<!\.)\z';
    my $START_TOKENS = '^|[\s:\[]';
    my $END_TOKENS = '[\s:\.\]]|$';

    # Replace all occurrences that are neither preceded nor followed by
    # numeric including hex digits, using negative lookbehind and negative
    # lookahead. Might be better to do this instead of the start and end
    # tokens, as well.
    if ($macro_value =~ /^$IPv4_REGEXP$|^$IPv6_REGEXP$/i) {
	if ($line =~ /$START_TOKENS$macro_value$END_TOKENS/) {	
	    if ($append) {
		$line =~ s/(?<![\da-f])($macro_value)(?![\da-f])/$1\[$macro_name\]/g;
	    }
	    else {
		$line =~ s/(?<![\da-f])($macro_value)(?![\da-f])/$macro_name/g;		
	    }
	}
    }
    else {
	if ($append) {
	    $line =~ s/($macro_value)/$1\[$macro_name\]/g;
	}
	else {
	    $line =~ s/$macro_value/$macro_name/g;	
	}
    }
    return ($line);
}
