/*
 *  get_file_names.c - Part of AFD, an automatic file distribution program.
 *  Copyright (c) 1996 - 2022 Holger Kiehl <Holger.Kiehl@dwd.de>
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include "afddefs.h"

DESCR__S_M3
/*
 ** NAME
 **   get_file_names - gets the name of all files in a directory
 **
 ** SYNOPSIS
 **   int get_file_names(char *file_path, off_t *file_size_to_send)
 **
 ** DESCRIPTION
 **
 ** RETURN VALUES
 **   The number of files, total file size it has found in the directory
 **   and the directory where the files have been found. If all files
 **   are deleted due to age limit, it will return -1. Otherwise if
 **   an error occurs it will exit.
 **
 ** AUTHOR
 **   H.Kiehl
 **
 ** HISTORY
 **   31.12.1996 H.Kiehl Created
 **   31.01.1997 H.Kiehl Support for age limit.
 **   14.10.1998 H.Kiehl Free unused memory.
 **   03.02.2001 H.Kiehl Sort the files by date.
 **   21.08.2001 H.Kiehl Return -1 if all files have been deleted due
 **                      to age limit.
 **   20.12.2004 H.Kiehl Addition of delete file name buffer.
 **   14.02.2009 H.Kiehl Log to OUTPUT_LOG when we delete a file.
 **
 */
DESCR__E_M3

#include <stdio.h>                     /* NULL                           */
#include <string.h>                    /* strcpy(), strcat(), strcmp(),  */
                                       /* strerror()                     */
#include <stdlib.h>                    /* realloc(), malloc(), free()    */
#include <time.h>                      /* time()                         */
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>                    /* opendir(), closedir(), DIR,    */
                                       /* readdir(), struct dirent       */
#include <fcntl.h>
#include <unistd.h>                    /* unlink(), getpid()             */
#include <errno.h>
#include "fddefs.h"

/* Global variables. */
extern int                        counter_fd,
                                  files_to_delete,
                                  fsa_fd,
                                  *unique_counter;
#ifdef HAVE_HW_CRC32
extern int                        have_hw_crc32;
#endif
#ifdef _OUTPUT_LOG
extern int                        ol_fd;
# ifdef WITHOUT_FIFO_RW_SUPPORT
extern int                        ol_readfd;
# endif
extern unsigned int               *ol_job_number,
                                  *ol_retries;
extern char                       *ol_data,
                                  *ol_file_name,
                                  *ol_output_type;
extern unsigned short             *ol_archive_name_length,
                                  *ol_file_name_length,
                                  *ol_unl;
extern off_t                      *ol_file_size;
extern size_t                     ol_size,
                                  ol_real_size;
extern clock_t                    *ol_transfer_time;
#endif
extern off_t                      *file_size_buffer;
extern time_t                     *file_mtime_buffer;
extern char                       *p_work_dir,
                                  tr_hostname[],
                                  *del_file_name_buffer,
                                  *file_name_buffer;
extern struct filetransfer_status *fsa;
extern struct job                 db;
#ifdef WITH_DUP_CHECK
extern struct rule                *rule;
#endif
#ifdef _DELETE_LOG
extern struct delete_log          dl;
#endif

#if defined (_DELETE_LOG) || defined (_OUTPUT_LOG)
/* Local function prototypes. */
static void                       log_data(char *,
#ifdef HAVE_STATX
                                           struct statx *,
#else
                                           struct stat *,
#endif
# ifdef WITH_DUP_CHECK
                                           int,
# endif
# ifdef _OUTPUT_LOG
                                           char,
# endif
                                           time_t);
#endif


/*########################## get_file_names() ###########################*/
int
get_file_names(char *file_path, off_t *file_size_to_send)
{
#ifdef WITH_DUP_CHECK
   int           dup_counter = 0;
   off_t         dup_counter_size = 0;
#endif
   time_t        diff_time,
                 now;
   int           files_not_send = 0,
                 files_to_send = 0,
                 new_size,
                 offset;
   off_t         file_size_not_send = 0,
                 *p_file_size;
   time_t        *p_file_mtime;
   char          fullname[MAX_PATH_LENGTH],
                 *p_del_file_name,
                 *p_file_name,
                 *p_source_file;
#ifdef HAVE_STATX
   struct statx  stat_buf;
#else
   struct stat   stat_buf;
#endif
   struct dirent *p_dir;
   DIR           *dp;

   /*
    * Create directory name in which we can find the files for this job.
    */
   (void)strcpy(file_path, p_work_dir);
   (void)strcat(file_path, AFD_FILE_DIR);
   (void)strcat(file_path, OUTGOING_DIR);
   (void)strcat(file_path, "/");
   (void)strcat(file_path, db.msg_name);
   db.p_unique_name = db.msg_name;
#ifdef MULTI_FS_SUPPORT
   while ((*db.p_unique_name != '/') && (*db.p_unique_name != '\0'))
   {
      db.p_unique_name++; /* Away with the filesystem ID. */
   }
   if (*db.p_unique_name == '/')
   {
      db.p_unique_name++;
#endif
      while ((*db.p_unique_name != '/') && (*db.p_unique_name != '\0'))
      {
         db.p_unique_name++; /* Away with the job ID. */
      }
      if (*db.p_unique_name == '/')
      {
         db.p_unique_name++;
         while ((*db.p_unique_name != '/') && (*db.p_unique_name != '\0'))
         {
            db.p_unique_name++; /* Away with the dir number. */
         }
         if (*db.p_unique_name == '/')
         {
            int  i;
            char str_num[MAX_TIME_T_HEX_LENGTH + 1];

            db.p_unique_name++;
            i = 0;
            while ((i < MAX_TIME_T_HEX_LENGTH) &&
                   (*(db.p_unique_name + i) != '_') &&
                   (*(db.p_unique_name + i) != '\0'))
            {
               str_num[i] = *(db.p_unique_name + i);
               i++;
            }
            if ((*(db.p_unique_name + i) != '_') || (i == 0))
            {
               system_log(ERROR_SIGN, __FILE__, __LINE__,
                          "Could not determine message name from `%s'. #%x",
                          db.msg_name, db.id.job);
               exit(SYNTAX_ERROR);
            }
            str_num[i] = '\0';
            db.creation_time = (time_t)str2timet(str_num, NULL, 16);
            db.unl = i + 1;
            i = 0;
            while ((i < MAX_INT_HEX_LENGTH) &&
                   (*(db.p_unique_name + db.unl + i) != '_') &&
                   (*(db.p_unique_name + db.unl + i) != '\0'))
            {
               str_num[i] = *(db.p_unique_name + db.unl + i);
               i++;
            }
            if ((*(db.p_unique_name + db.unl + i) != '_') || (i == 0))
            {
               system_log(ERROR_SIGN, __FILE__, __LINE__,
                          "Could not determine message name from `%s'. #%x",
                          db.msg_name, db.id.job);
               exit(SYNTAX_ERROR);
            }
            str_num[i] = '\0';
            db.unique_number = (unsigned int)strtoul(str_num, NULL, 16);
            db.unl = db.unl + i + 1;
            i = 0;
            while ((i < MAX_INT_HEX_LENGTH) &&
                   (*(db.p_unique_name + db.unl + i) != '\0'))
            {
               str_num[i] = *(db.p_unique_name + db.unl + i);
               i++;
            }
            if (i == 0)
            {
               system_log(ERROR_SIGN, __FILE__, __LINE__,
                          "Could not determine message name from `%s'. #%x",
                          db.msg_name, db.id.job);
               exit(SYNTAX_ERROR);
            }
            str_num[i] = '\0';
            db.split_job_counter = (unsigned int)strtoul(str_num, NULL, 16);
            db.unl = db.unl + i;
            if (db.unl == 0)
            {
               system_log(ERROR_SIGN, __FILE__, __LINE__,
                          "Could not determine message name from `%s'. #%x",
                          db.msg_name, db.id.job);
               exit(SYNTAX_ERROR);
            }
         }
         else
         {
            system_log(ERROR_SIGN, __FILE__, __LINE__,
                       "Could not determine message name from `%s'. #%x",
                       db.msg_name, db.id.job);
            exit(SYNTAX_ERROR);
         }
      }
      else
      {
         system_log(ERROR_SIGN, __FILE__, __LINE__,
                    "Could not determine message name from `%s'. #%x",
                    db.msg_name, db.id.job);
         exit(SYNTAX_ERROR);
      }
#ifdef MULTI_FS_SUPPORT
   }
   else
   {
      system_log(ERROR_SIGN, __FILE__, __LINE__,
                 "Could not determine message name from `%s'. #%x",
                 db.msg_name, db.id.job);
      exit(SYNTAX_ERROR);
   }
#endif

   if ((dp = opendir(file_path)) == NULL)
   {
      system_log((errno != ENOENT) ? ERROR_SIGN : WARN_SIGN, __FILE__, __LINE__,
                 "Could not opendir() %s [%s %d] : %s #%x",
                 file_path, db.host_alias, (int)db.job_no, strerror(errno),
                 db.id.job);
      exit(OPEN_FILE_DIR_ERROR);
   }

   /* Prepare pointers and directory name. */
   (void)strcpy(fullname, file_path);
   p_source_file = fullname + strlen(fullname);
   *p_source_file++ = '/';
   if (file_name_buffer != NULL)
   {
      free(file_name_buffer);
      file_name_buffer = NULL;
   }
   p_file_name = file_name_buffer;
   if (file_size_buffer != NULL)
   {
      free(file_size_buffer);
      file_size_buffer = NULL;
   }
   p_file_size = file_size_buffer;
   if (file_mtime_buffer != NULL)
   {
      free(file_mtime_buffer);
      file_mtime_buffer = NULL;
   }
   p_file_mtime = file_mtime_buffer;
   if (del_file_name_buffer != NULL)
   {
      free(del_file_name_buffer);
      del_file_name_buffer = NULL;
   }
   p_del_file_name = del_file_name_buffer;
   files_to_delete = 0;
   now = time(NULL);

   /*
    * Now let's determine the number of files that have to be
    * transmitted and the total size.
    */
   errno = 0;
   while ((p_dir = readdir(dp)) != NULL)
   {
      if (
#ifdef LINUX
          (p_dir->d_type != DT_REG) ||
#endif
          ((p_dir->d_name[0] == '.') && (p_dir->d_name[1] == '\0')) ||
          ((p_dir->d_name[0] == '.') && (p_dir->d_name[1] == '.') &&
          (p_dir->d_name[2] == '\0')))
      {
         continue;
      }
      (void)strcpy(p_source_file, p_dir->d_name);
#ifdef HAVE_STATX
      if (statx(0, fullname, AT_STATX_SYNC_AS_STAT,
# ifndef LINUX
                STATX_MODE |
# endif
                STATX_SIZE | STATX_MTIME, &stat_buf) == -1)
#else
      if (stat(fullname, &stat_buf) == -1)
#endif
      {
         system_log(WARN_SIGN, __FILE__, __LINE__,
#ifdef HAVE_STATX
                    "Can't statx() file `%s' : %s #%x",
#else
                    "Can't stat() file `%s' : %s #%x",
#endif
                    fullname, strerror(errno), db.id.job);
         continue;
      }

#ifndef LINUX
      /* Sure it is a normal file? */
# ifdef HAVE_STATX
      if (S_ISREG(stat_buf.stx_mode))
# else
      if (S_ISREG(stat_buf.st_mode))
# endif
#endif
      {
#ifdef WITH_DUP_CHECK
         int is_duplicate = NO;
#endif
         int remove_file = NO;

#ifdef HAVE_STATX
         if (now < stat_buf.stx_mtime.tv_sec)
#else
         if (now < stat_buf.st_mtime)
#endif
         {
            diff_time = 0L;
         }
         else
         {
#ifdef HAVE_STATX
            diff_time = now - stat_buf.stx_mtime.tv_sec;
#else
            diff_time = now - stat_buf.st_mtime;
#endif
         }

         /* Don't send files older then age_limit! */
         if ((db.age_limit > 0) &&
             ((fsa->host_status & DO_NOT_DELETE_DATA) == 0) &&
             (diff_time > db.age_limit))
         {
            remove_file = YES;
         }
         else
         {
#ifdef WITH_DUP_CHECK
            if (((db.age_limit > 0) &&
                 ((fsa->host_status & DO_NOT_DELETE_DATA) == 0) &&
                 (diff_time > db.age_limit)) ||
                ((db.dup_check_timeout > 0) &&
# ifdef FAST_SF_DUPCHECK
                 ((db.special_flag & OLD_ERROR_JOB) == 0) &&
                 (((is_duplicate = isdup(fullname, p_dir->d_name,
#  ifdef HAVE_STATX
                                         stat_buf.stx_size,
#  else
                                         stat_buf.st_size,
#  endif
                                         db.crc_id,
                                         db.dup_check_timeout,
                                         db.dup_check_flag,
                                         NO,
#  ifdef HAVE_HW_CRC32
                                         have_hw_crc32,
#  endif
                                         YES, YES)) == YES) &&
                  ((db.dup_check_flag & DC_DELETE) ||
                   (db.dup_check_flag & DC_STORE)))
# else
                 (((is_duplicate = isdup(fullname, p_dir->d_name,
#  ifdef HAVE_STATX
                                         stat_buf.stx_size,
#  else
                                         stat_buf.st_size,
#  endif
                                         db.crc_id,
                                         0,
                                         db.dup_check_flag,
                                         YES,
#  ifdef HAVE_HW_CRC32
                                         have_hw_crc32,
#  endif
                                         YES, YES)) == YES) &&
                  ((db.dup_check_flag & DC_DELETE) ||
                   (db.dup_check_flag & DC_STORE)))
# endif
                ))
             {
                remove_file = YES;
             }
             else
             {
                if ((db.trans_dup_check_timeout > 0) &&
                    ((db.special_flag & OLD_ERROR_JOB) == 0))
                {
                   register int k;
                   char         tmp_filename[MAX_PATH_LENGTH];

                   tmp_filename[0] = '\0';
                   for (k = 0; k < rule[db.trans_rule_pos].no_of_rules; k++)
                   {
                      if (pmatch(rule[db.trans_rule_pos].filter[k],
                                 p_dir->d_name, NULL) == 0)
                      {
                         change_name(p_dir->d_name,
                                     rule[db.trans_rule_pos].filter[k],
                                     rule[db.trans_rule_pos].rename_to[k],
                                     tmp_filename, MAX_PATH_LENGTH,
                                     &counter_fd, &unique_counter, db.id.job);
                         break;
                      }
                   }
                   if (tmp_filename[0] != '\0')
                   {
                      if (((is_duplicate = isdup(fullname, tmp_filename,
# ifdef HAVE_STATX
                                                 stat_buf.stx_size,
# else
                                                 stat_buf.st_size,
# endif
                                                 db.crc_id,
                                                 db.trans_dup_check_timeout,
                                                 db.trans_dup_check_flag,
                                                 NO,
# ifdef HAVE_HW_CRC32
                                                 have_hw_crc32,
# endif
                                                 YES, YES)) == YES) &&
                          ((db.trans_dup_check_flag & DC_DELETE) ||
                           (db.trans_dup_check_flag & DC_STORE)))
                      {
                         remove_file = YES;
                      }
                   }
                }
             }
#endif
         }

         if (remove_file == YES)
         {
            char file_to_remove[MAX_FILENAME_LENGTH];

            file_to_remove[0] = '\0';
            if (db.no_of_restart_files > 0)
            {
               int  ii;
               char initial_filename[MAX_FILENAME_LENGTH];

               if ((db.lock == DOT) || (db.lock == DOT_VMS))
               {
                  (void)strcpy(initial_filename, db.lock_notation);
                  (void)strcat(initial_filename, p_dir->d_name);
               }
               else
               {
                  (void)strcpy(initial_filename, p_dir->d_name);
               }

               for (ii = 0; ii < db.no_of_restart_files; ii++)
               {
                  if ((CHECK_STRCMP(db.restart_file[ii], initial_filename) == 0) &&
                      (append_compare(db.restart_file[ii], fullname) == YES))
                  {
                     (void)strcpy(file_to_remove, db.restart_file[ii]);
                     remove_append(db.id.job, db.restart_file[ii]);
                     break;
                  }
               }
            }
#ifdef WITH_DUP_CHECK
            if (is_duplicate == YES)
            {
               dup_counter++;
# ifdef HAVE_STATX
               dup_counter_size += stat_buf.stx_size;
# else
               dup_counter_size += stat_buf.st_size;
# endif
               if (db.dup_check_flag & DC_WARN)
               {
                  trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, NULL,
                            "File `%s' is duplicate. #%x",
                            p_dir->d_name, db.id.job);
               }
            }
            if ((is_duplicate == YES) && (db.dup_check_flag & DC_STORE))
            {
               char *p_end,
                    save_dir[MAX_PATH_LENGTH];

               p_end = save_dir +
                       snprintf(save_dir, MAX_PATH_LENGTH, "%s%s%s/%x/",
                                p_work_dir, AFD_FILE_DIR, STORE_DIR, db.id.job);
               if ((mkdir(save_dir, DIR_MODE) == -1) && (errno != EEXIST))
               {
                  system_log(WARN_SIGN, __FILE__, __LINE__,
                             "Failed to mkdir() `%s' : %s",
                             save_dir, strerror(errno));
                  if (unlink(fullname) == -1)
                  {
                     system_log(WARN_SIGN, __FILE__, __LINE__,
                                "Failed to unlink() file `%s' due to duplicate check : %s #%x",
                                p_dir->d_name, strerror(errno), db.id.job);
                  }
# if defined (_DELETE_LOG) || defined (_OUTPUT_LOG)
                  else
                  {
                     log_data(p_dir->d_name, &stat_buf, YES,
#  ifdef _OUTPUT_LOG
                              OT_DUPLICATE_DELETE + '0',
#  endif
                              now);
                  }
# endif
               }
               else
               {
                  (void)strcpy(p_end, p_dir->d_name);
                  if (rename(fullname, save_dir) == -1)
                  {
                     system_log(WARN_SIGN, __FILE__, __LINE__,
                                "Failed to rename() `%s' to `%s' : %s #%x",
                                fullname, save_dir, strerror(errno), db.id.job);
                     if (unlink(fullname) == -1)
                     {
                        system_log(WARN_SIGN, __FILE__, __LINE__,
                                   "Failed to unlink() file `%s' due to duplicate check : %s #%x",
                                   p_dir->d_name, strerror(errno), db.id.job);
                     }
# if defined (_DELETE_LOG) || defined (_OUTPUT_LOG)
                     else
                     {
                        log_data(p_dir->d_name, &stat_buf, YES,
#  ifdef _OUTPUT_LOG
                                 OT_DUPLICATE_STORED + '0',
#  endif
                                 now);
                     }
# endif
                  }
               }
               files_not_send++;
# ifdef HAVE_STATX
               file_size_not_send += stat_buf.stx_size;
# else
               file_size_not_send += stat_buf.st_size;
# endif
            }
            else
            {
#endif /* WITH_DUP_CHECK */
               if (unlink(fullname) == -1)
               {
                  system_log(ERROR_SIGN, __FILE__, __LINE__,
                             "Failed to unlink() file `%s' due to age : %s #%x",
                             p_dir->d_name, strerror(errno), db.id.job);
               }
               else
               {
#if defined (_DELETE_LOG) || defined (_OUTPUT_LOG)
                  log_data(p_dir->d_name, &stat_buf,
# ifdef WITH_DUP_CHECK
                           is_duplicate,
#  ifdef _OUTPUT_LOG
                           (is_duplicate == YES) ? (OT_DUPLICATE_DELETE + '0') : (OT_AGE_LIMIT_DELETE + '0'),
#  endif
# else
#  ifdef _OUTPUT_LOG
                           OT_AGE_LIMIT_DELETE + '0',
#  endif
# endif
                           now);
#endif
#ifndef _DELETE_LOG
                  if ((db.protocol & FTP_FLAG) || (db.protocol & SFTP_FLAG))
                  {
                     if (db.no_of_restart_files > 0)
                     {
                        int append_file_number = -1,
                            ii;

                        for (ii = 0; ii < db.no_of_restart_files; ii++)
                        {
                           if (CHECK_STRCMP(db.restart_file[ii], p_dir->d_name) == 0)
                           {
                              append_file_number = ii;
                              break;
                           }
                        }
                        if (append_file_number != -1)
                        {
                           remove_append(db.id.job,
                                         db.restart_file[append_file_number]);
                        }
                     }
                  }
#endif
                  if (file_to_remove[0] != '\0')
                  {
                     if ((files_to_delete % 20) == 0)
                     {
                        /* Increase the space for the delete file name buffer. */
                        new_size = ((files_to_delete / 20) + 1) *
                                   20 * MAX_FILENAME_LENGTH;
                        offset = p_del_file_name - del_file_name_buffer;
                        if ((del_file_name_buffer = realloc(del_file_name_buffer,
                                                            new_size)) == NULL)
                        {
                           system_log(ERROR_SIGN, __FILE__, __LINE__,
                                      "Could not realloc() memory : %s #%x",
                                      strerror(errno), db.id.job);
                           exit(ALLOC_ERROR);
                        }
                        p_del_file_name = del_file_name_buffer + offset;
                     }
                     (void)strcpy(p_del_file_name, file_to_remove);
                     p_del_file_name += MAX_FILENAME_LENGTH;
                     files_to_delete++;
                  }

                  files_not_send++;
#ifdef HAVE_STATX
                  file_size_not_send += stat_buf.stx_size;
#else
                  file_size_not_send += stat_buf.st_size;
#endif
               }
#ifdef WITH_DUP_CHECK
            }
#endif
         }
         else
         {
#ifdef WITH_DUP_CHECK
            if ((is_duplicate == YES) && (db.dup_check_flag & DC_WARN))
            {
               trans_log(WARN_SIGN, __FILE__, __LINE__, NULL, NULL,
                         "File `%s' is duplicate. #%x", p_dir->d_name, db.id.job);
            }
#endif
            if ((files_to_send % 20) == 0)
            {
               /* Increase the space for the file name buffer. */
               new_size = ((files_to_send / 20) + 1) *
                          20 * MAX_FILENAME_LENGTH;
               offset = p_file_name - file_name_buffer;
               if ((file_name_buffer = realloc(file_name_buffer,
                                               new_size)) == NULL)
               {
                  system_log(ERROR_SIGN, __FILE__, __LINE__,
                             "Could not realloc() memory : %s #%x",
                             strerror(errno), db.id.job);
                  exit(ALLOC_ERROR);
               }
               p_file_name = file_name_buffer + offset;

               /* Increase the space for the file size buffer. */
               new_size = ((files_to_send / 20) + 1) * 20 * sizeof(off_t);
               offset = (char *)p_file_size - (char *)file_size_buffer;
               if ((file_size_buffer = realloc(file_size_buffer,
                                               new_size)) == NULL)
               {
                  system_log(ERROR_SIGN, __FILE__, __LINE__,
                             "Could not realloc() memory : %s #%x",
                             strerror(errno), db.id.job);
                  exit(ALLOC_ERROR);
               }
               p_file_size = (off_t *)((char *)file_size_buffer + offset);

               if ((fsa->protocol_options & SORT_FILE_NAMES) ||
#ifdef WITH_EUMETSAT_HEADERS
                   (db.special_flag & ADD_EUMETSAT_HEADER) ||
#endif
                   (fsa->protocol_options & KEEP_TIME_STAMP))
               {
                  /* Increase the space for the file mtime buffer. */
                  new_size = ((files_to_send / 20) + 1) * 20 * sizeof(time_t);
                  offset = (char *)p_file_mtime - (char *)file_mtime_buffer;
                  if ((file_mtime_buffer = realloc(file_mtime_buffer,
                                                   new_size)) == NULL)
                  {
                     system_log(ERROR_SIGN, __FILE__, __LINE__,
                                "Could not realloc() memory : %s #%x",
                                strerror(errno), db.id.job);
                     exit(ALLOC_ERROR);
                  }
                  p_file_mtime = (time_t *)((char *)file_mtime_buffer + offset);
               }
            }

            /* Sort the files, newest must be last (FIFO). */
            if ((fsa->protocol_options & SORT_FILE_NAMES) &&
                (files_to_send > 1) &&
#ifdef HAVE_STATX
                (*(p_file_mtime - 1) > stat_buf.stx_mtime.tv_sec)
#else
                (*(p_file_mtime - 1) > stat_buf.st_mtime)
#endif
               )
            {
               int    i;
               off_t  *sp_file_size = p_file_size - 1;
               time_t *sp_mtime = p_file_mtime - 1;
               char   *sp_file_name = p_file_name - MAX_FILENAME_LENGTH;

               for (i = files_to_send; i > 0; i--)
               {
#ifdef HAVE_STATX
                  if (*sp_mtime <= stat_buf.stx_mtime.tv_sec)
#else
                  if (*sp_mtime <= stat_buf.st_mtime)
#endif
                  {
                     break;
                  }
                  sp_mtime--; sp_file_size--;
                  sp_file_name -= MAX_FILENAME_LENGTH;
               }
               (void)memmove(sp_mtime + 2, sp_mtime + 1,
                             (files_to_send - i) * sizeof(time_t));
#ifdef HAVE_STATX
               *(sp_mtime + 1) = stat_buf.stx_mtime.tv_sec;
#else
               *(sp_mtime + 1) = stat_buf.st_mtime;
#endif
               (void)memmove(sp_file_size + 2, sp_file_size + 1,
                             (files_to_send - i) * sizeof(off_t));
#ifdef HAVE_STATX
               *(sp_file_size + 1) = stat_buf.stx_size;
#else
               *(sp_file_size + 1) = stat_buf.st_size;
#endif
               (void)memmove(sp_file_name + (2 * MAX_FILENAME_LENGTH),
                             sp_file_name + MAX_FILENAME_LENGTH,
                             (files_to_send - i) * MAX_FILENAME_LENGTH);
               (void)strcpy(sp_file_name + MAX_FILENAME_LENGTH,
                            p_dir->d_name);
            }
            else
            {
               (void)strcpy(p_file_name, p_dir->d_name);
#ifdef HAVE_STATX
               *p_file_size = stat_buf.stx_size;
#else
               *p_file_size = stat_buf.st_size;
#endif
               if (file_mtime_buffer != NULL)
               {
#ifdef HAVE_STATX
                  *p_file_mtime = stat_buf.stx_mtime.tv_sec;
#else
                  *p_file_mtime = stat_buf.st_mtime;
#endif
               }
            }
            p_file_name += MAX_FILENAME_LENGTH;
            p_file_size++;
            p_file_mtime++;
            files_to_send++;
#ifdef HAVE_STATX
            *file_size_to_send += stat_buf.stx_size;
#else
            *file_size_to_send += stat_buf.st_size;
#endif
         }
      }
      errno = 0;
   } /* while ((p_dir = readdir(dp)) != NULL) */

   if (errno)
   {
      system_log(ERROR_SIGN, __FILE__, __LINE__,
                 "Could not readdir() `%s' : %s #%x",
                 file_path, strerror(errno), db.id.job);
   }

#ifdef WITH_DUP_CHECK
   isdup_detach();
#endif

   if ((file_mtime_buffer != NULL) &&
#ifdef WITH_EUMETSAT_HEADERS
       ((db.special_flag & ADD_EUMETSAT_HEADER) == 0) &&
#endif
       ((fsa->protocol_options & KEEP_TIME_STAMP) == 0))
   {
      free(file_mtime_buffer);
      file_mtime_buffer = NULL;
   }

   if (files_not_send > 0)
   {
      /* Total file counter. */
#ifdef LOCK_DEBUG
      lock_region_w(fsa_fd, db.lock_offset + LOCK_TFC, __FILE__, __LINE__);
#else
      lock_region_w(fsa_fd, db.lock_offset + LOCK_TFC);
#endif
      fsa->total_file_counter -= files_not_send;
#ifdef _VERIFY_FSA
      if (fsa->total_file_counter < 0)
      {
         system_log(DEBUG_SIGN, __FILE__, __LINE__,
                    "Total file counter for host `%s' less then zero. Correcting to 0. #%x",
                    fsa->host_dsp_name, db.id.job);
         fsa->total_file_counter = 0;
      }
#endif

      /* Total file size. */
      fsa->total_file_size -= file_size_not_send;
#ifdef _VERIFY_FSA
      if (fsa->total_file_size < 0)
      {
         system_log(DEBUG_SIGN, __FILE__, __LINE__,
                    "Total file size for host `%s' overflowed. Correcting to 0. #%x",
                    fsa->host_dsp_name, db.id.job);
         fsa->total_file_size = 0;
      }
      else if ((fsa->total_file_counter == 0) &&
               (fsa->total_file_size > 0))
           {
              system_log(DEBUG_SIGN, __FILE__, __LINE__,
                         "fc for host `%s' is zero but fs is not zero. Correcting to 0. #%x",
                         fsa->host_dsp_name, db.id.job);
              fsa->total_file_size = 0;
           }
#endif

      if ((fsa->total_file_counter == 0) && (fsa->total_file_size == 0) &&
          (fsa->error_counter > 0))
      {
         int j;

#ifdef LOCK_DEBUG
         lock_region_w(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__);
#else
         lock_region_w(fsa_fd, db.lock_offset + LOCK_EC);
#endif
         fsa->error_counter = 0;

         /*
          * Remove the error condition (NOT_WORKING) from all jobs
          * of this host.
          */
         for (j = 0; j < fsa->allowed_transfers; j++)
         {
            if ((j != db.job_no) &&
                (fsa->job_status[j].connect_status == NOT_WORKING))
            {
               fsa->job_status[j].connect_status = DISCONNECT;
            }
         }
         fsa->error_history[0] = 0;
         fsa->error_history[1] = 0;
#ifdef LOCK_DEBUG
         unlock_region(fsa_fd, db.lock_offset + LOCK_EC, __FILE__, __LINE__);
#else
         unlock_region(fsa_fd, db.lock_offset + LOCK_EC);
#endif
      }
#ifdef LOCK_DEBUG
      unlock_region(fsa_fd, db.lock_offset + LOCK_TFC, __FILE__, __LINE__);
#else
      unlock_region(fsa_fd, db.lock_offset + LOCK_TFC);
#endif
#ifdef WITH_DUP_CHECK
      if (dup_counter > 0)
      {
         trans_log(INFO_SIGN, NULL, 0, NULL, NULL,
# if SIZEOF_OFF_T == 4
                   "Deleted %d duplicate file(s) (%ld bytes). #%x",
# else
                   "Deleted %d duplicate file(s) (%lld bytes). #%x",
# endif
                   dup_counter, (pri_off_t)dup_counter_size, db.id.job);
      }
      if ((files_not_send - dup_counter) > 0)
      {
         trans_log(INFO_SIGN, NULL, 0, NULL, NULL,
# if SIZEOF_OFF_T == 4
                   "Deleted %d file(s) (%ld bytes) due to age. #%x",
# else
                   "Deleted %d file(s) (%lld bytes) due to age. #%x",
# endif
                   files_not_send - dup_counter,
                   (pri_off_t)(file_size_not_send - dup_counter_size),
                   db.id.job);
      }
#else
      trans_log(INFO_SIGN, NULL, 0, NULL, NULL,
# if SIZEOF_OFF_T == 4
                "Deleted %d file(s) (%ld bytes) due to age. #%x",
# else
                "Deleted %d file(s) (%lld bytes) due to age. #%x",
# endif
                files_not_send, (pri_off_t)file_size_not_send,
                db.id.job);
#endif
#ifdef WITH_ERROR_QUEUE
      if ((files_to_send == 0) && (fsa->host_status & ERROR_QUEUE_SET) &&
          (check_error_queue(db.id.job, -1, 0, 0) == YES))
      {
         (void)remove_from_error_queue(db.id.job, fsa, db.fsa_pos, fsa_fd);
      }
#endif
   }

   if (closedir(dp) < 0)
   {
      system_log(ERROR_SIGN, __FILE__, __LINE__,
                 "Could not closedir() `%s' : %s #%x",
                 file_path, strerror(errno), db.id.job);
      exit(OPEN_FILE_DIR_ERROR);
   }

   /*
    * If we just return zero here when all files have been deleted
    * due to age and sf_xxx is bursting it does not know if this
    * is an error situation or not. So return -1 if we deleted all
    * files.
    */
   if ((files_to_send == 0) && (files_not_send > 0))
   {
      files_to_send = -1;
   }
   return(files_to_send);
}


#if defined (_DELETE_LOG) || defined (_OUTPUT_LOG)
/*++++++++++++++++++++++++++++++ log_data() +++++++++++++++++++++++++++++*/
static void
log_data(char         *d_name,
#ifdef HAVE_STATX
         struct statx *stat_buf,
#else
         struct stat  *stat_buf,
#endif
# ifdef WITH_DUP_CHECK
         int          is_duplicate,
# endif
# ifdef _OUTPUT_LOG
         char         output_type,
# endif
         time_t       now)
{
# ifdef _DELETE_LOG
   int    prog_name_length;
   size_t dl_real_size;
   char   str_diff_time[2 + MAX_LONG_LENGTH + 6 + MAX_LONG_LENGTH + 12 + MAX_LONG_LENGTH + 30 + 1];
# endif

# ifdef _OUTPUT_LOG
   if (db.output_log == YES)
   {
      if (ol_fd == -2)
      {
#  ifdef WITHOUT_FIFO_RW_SUPPORT
         output_log_fd(&ol_fd, &ol_readfd, &db.output_log);
#  else
         output_log_fd(&ol_fd, &db.output_log);
#  endif
      }
      if (ol_fd > -1)
      {
         if (ol_data == NULL)
         {
            int current_toggle,
                protocol;

            if (db.protocol & FTP_FLAG)
            {
#  ifdef WITH_SSL
               if (db.tls_auth == NO)
               {
                  protocol = FTP;
               }
               else
               {
                  protocol = FTPS;
               }
#  else
               protocol = FTP;
#  endif
            }
            else if (db.protocol & LOC_FLAG)
                 {
                    protocol = LOC;
                 }
            else if (db.protocol & EXEC_FLAG)
                 {
                    protocol = EXEC;
                 }
            else if (db.protocol & HTTP_FLAG)
                 {
#  ifdef WITH_SSL
                    if (db.tls_auth == NO)
                    {
                       protocol = HTTP;
                    }
                    else
                    {
                       protocol = HTTPS;
                    }
#  else
                    protocol = HTTP;
#  endif
                 }
            else if (db.protocol & SFTP_FLAG)
                 {
                    protocol = SFTP;
                 }
#  ifdef _WITH_SCP_SUPPORT
            else if (db.protocol & SCP_FLAG)
                 {
                    protocol = SCP;
                 }
#  endif
#  ifdef _WITH_WMO_SUPPORT
            else if (db.protocol & WMO_FLAG)
                 {
                    protocol = WMO;
                 }
#  endif
#  ifdef _WITH_MAP_SUPPORT
            else if (db.protocol & MAP_FLAG)
                 {
                    protocol = MAP;
                 }
#  endif
#  ifdef _WITH_DFAX_SUPPORT
            else if (db.protocol & DFAX_FLAG)
                 {
                    protocol = DFAX;
                 }
#  endif
            else if (db.protocol & SMTP_FLAG)
                 {
                    protocol = SMTP;
                 }
#  ifdef _WITH_DE_MAIL_SUPPORT
            else if (db.protocol & DE_MAIL_FLAG)
                 {
                    protocol = DE_MAIL;
                 }
#  endif
                 else
                 {
                    system_log(DEBUG_SIGN, __FILE__, __LINE__,
                               "Unknown protocol flag %d, setting FTP. #%x",
                               db.protocol, db.id.job);
                    protocol = FTP;
                 }

            if (fsa->real_hostname[1][0] == '\0')
            {
               current_toggle = HOST_ONE;
            }
            else
            {
               if (db.toggle_host == YES)
               {
                  if (fsa->host_toggle == HOST_ONE)
                  {
                     current_toggle = HOST_TWO;
                  }
                  else
                  {
                     current_toggle = HOST_ONE;
                  }
               }
               else
               {
                  current_toggle = (int)fsa->host_toggle;
               }
            }

            output_log_ptrs(&ol_retries,
                            &ol_job_number,
                            &ol_data,   /* Pointer to buffer. */
                            &ol_file_name,
                            &ol_file_name_length,
                            &ol_archive_name_length,
                            &ol_file_size,
                            &ol_unl,
                            &ol_size,
                            &ol_transfer_time,
                            &ol_output_type,
                            db.host_alias,
                            (current_toggle - 1),
                            protocol,
                            &db.output_log);
         }
         (void)memcpy(ol_file_name, db.p_unique_name, db.unl);
         (void)strcpy(ol_file_name + db.unl, d_name);
         *ol_file_name_length = (unsigned short)strlen(ol_file_name);
         ol_file_name[*ol_file_name_length] = SEPARATOR_CHAR;
         ol_file_name[*ol_file_name_length + 1] = '\0';
         (*ol_file_name_length)++;
#  ifdef HAVE_STATX
         *ol_file_size = stat_buf->stx_size;
#  else
         *ol_file_size = stat_buf->st_size;
#  endif
         *ol_job_number = db.id.job;
         *ol_retries = db.retries;
         *ol_unl = db.unl;
         *ol_transfer_time = 0L;
         *ol_archive_name_length = 0;
         *ol_output_type = output_type;
         ol_real_size = *ol_file_name_length + ol_size;
         if (write(ol_fd, ol_data, ol_real_size) != ol_real_size)
         {
            system_log(ERROR_SIGN, __FILE__, __LINE__,
                       "write() error : %s #%x", strerror(errno), db.id.job);
         }
      }
   }
# endif

# ifdef _DELETE_LOG
   if (dl.fd == -1)
   {
      delete_log_ptrs(&dl);
   }
   (void)strcpy(dl.file_name, d_name);
   (void)snprintf(dl.host_name, MAX_HOSTNAME_LENGTH + 4 + 1, "%-*s %03x",
                  MAX_HOSTNAME_LENGTH, fsa->host_alias,
#  ifdef WITH_DUP_CHECK
                  (is_duplicate == YES) ? DUP_OUTPUT : AGE_OUTPUT);
#  else
                  AGE_OUTPUT);
#  endif
#  ifdef HAVE_STATX
   *dl.file_size = stat_buf->stx_size;
#  else
   *dl.file_size = stat_buf->st_size;
#  endif
   *dl.job_id = db.id.job;
   *dl.dir_id = 0;
   *dl.input_time = db.creation_time;
   *dl.split_job_counter = db.split_job_counter;
   *dl.unique_number = db.unique_number;
   *dl.file_name_length = strlen(d_name);
#  ifdef WITH_DUP_CHECK
   if (is_duplicate == YES)
   {
      str_diff_time[0] = '\0';
   }
   else
   {
      time_t diff_time;

#   ifdef HAVE_STATX
      if (now < stat_buf->stx_mtime.tv_sec)
#   else
      if (now < stat_buf->st_mtime)
#   endif
      {
         diff_time = 0L;
      }
      else
      {
#   ifdef HAVE_STATX
         diff_time = now - stat_buf->stx_mtime.tv_sec;
#   else
         diff_time = now - stat_buf->st_mtime;
#   endif
      }
#  endif
      (void)snprintf(str_diff_time,
                     2 + MAX_LONG_LENGTH + 6 + MAX_LONG_LENGTH + 12 + MAX_LONG_LENGTH + 30 + 1,
#  if SIZEOF_TIME_T == 4
                     "%c>%ld [now=%ld file_mtime=%ld] (%s %d)",
#  else
                     "%c>%lld [now=%lld file_mtime=%lld] (%s %d)",
#  endif
                     SEPARATOR_CHAR, (pri_time_t)diff_time, (pri_time_t)now,
#  ifdef HAVE_STATX
                     (pri_time_t)stat_buf->stx_mtime.tv_sec,
#  else
                     (pri_time_t)stat_buf->st_mtime,
#  endif
                     __FILE__, __LINE__);
#  ifdef WITH_DUP_CHECK
   }
#  endif
   if (db.protocol & FTP_FLAG)
   {
      if (db.no_of_restart_files > 0)
      {
         int append_file_number = -1,
             ii;

         for (ii = 0; ii < db.no_of_restart_files; ii++)
         {
            if (CHECK_STRCMP(db.restart_file[ii], d_name) == 0)
            {
               append_file_number = ii;
               break;
            }
         }
         if (append_file_number != -1)
         {
            remove_append(db.id.job,
                          db.restart_file[append_file_number]);
         }
      }
      prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                  MAX_FILENAME_LENGTH + 1,
                                  "%s%s", SEND_FILE_FTP, str_diff_time);
   }
   else if (db.protocol & LOC_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_LOC, str_diff_time);
        }
   else if (db.protocol & EXEC_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_EXEC, str_diff_time);
        }
   else if (db.protocol & HTTP_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_HTTP, str_diff_time);
        }
   else if (db.protocol & SFTP_FLAG)
        {
           if (db.no_of_restart_files > 0)
           {
              int append_file_number = -1,
                  ii;

              for (ii = 0; ii < db.no_of_restart_files; ii++)
              {
                 if (CHECK_STRCMP(db.restart_file[ii], d_name) == 0)
                 {
                    append_file_number = ii;
                    break;
                 }
              }
              if (append_file_number != -1)
              {
                 remove_append(db.id.job,
                               db.restart_file[append_file_number]);
              }
           }
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_SFTP, str_diff_time);
        }
#  ifdef _WITH_SCP_SUPPORT
   else if (db.protocol & SCP_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_SCP, str_diff_time);
        }
#  endif
#  ifdef _WITH_WMO_SUPPORT
   else if (db.protocol & WMO_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_WMO, str_diff_time);
        }
#  endif
#  ifdef _WITH_MAP_SUPPORT
   else if (db.protocol & MAP_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_MAP, str_diff_time);
        }
#  endif
#  ifdef _WITH_DFAX_SUPPORT
   else if (db.protocol & DFAX_FLAG)
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_DFAX, str_diff_time);
        }
#  endif
#  ifdef _WITH_DE_MAIL_SUPPORT
   else if ((db.protocol & SMTP_FLAG) || (db.protocol & DE_MAIL_FLAG))
#  else
   else if (db.protocol & SMTP_FLAG)
#  endif
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "%s%s", SEND_FILE_SMTP, str_diff_time);
        }
        else
        {
           prog_name_length = snprintf((dl.file_name + *dl.file_name_length + 1),
                                       MAX_FILENAME_LENGTH + 1,
                                       "sf_???%s", str_diff_time);
        }

   dl_real_size = *dl.file_name_length + dl.size +
                  prog_name_length;
   if (write(dl.fd, dl.data, dl_real_size) != dl_real_size)
   {
      system_log(ERROR_SIGN, __FILE__, __LINE__,
                 "write() error : %s #%x", strerror(errno), db.id.job);
   }
# endif

   return;
}
#endif