The Java Program will either archive files or delete files that are older than a specified period (RetentionPeriodInDays).
List of folders to be purged / archived is configurable and is mentioned in a XML config file
If operation is 'archive', compress old files to *.tar.gz and move to Archive folder.
If operation is 'delete', delete files older than 'retention period'.
List of folders to be purged / archived is configurable and is mentioned in a XML config file
If operation is 'archive', compress old files to *.tar.gz and move to Archive folder.
If operation is 'delete', delete files older than 'retention period'.
package service;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;
import org.apache.commons.compress.archivers.ArchiveException;
import org.apache.commons.compress.archivers.ArchiveStreamFactory;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import directory.generated.Folder;
import directory.generated.Folders;
import directory.generated.OperationType;
/**
*
*
* IF OPERATION IS ARCHIVE, COMPRESS OLD FILES TO *.TAR.GZ AND MOVE TO
* ARCHIVE FOLDER.
*
* IF OPERATION IS DELETE, DELETE FILES OLDER THAN 'RETENTION PERIOD'.
*
* LIST OF FOLDERS TO BE PURGED/ARCHIVED IS MENTIONED IN A XML CONFIG
* FILE
*
*/
public class HouseKeeping {
// CONFIG FILE LOCATION, TODO GET FROM RESOURCE BUNDLE
private static final String configFile = "src/config/Sample.xml";
private Logger logger = LoggerFactory.getLogger(this.getClass()
.getSimpleName());
private Date currDate = new Date();
private String pathToBeExcluded = "";
/**
* Main method
*
* @throws Exception
*/
public void houseKeepingService() throws Exception {
logger.info("HOUSE KEEPING PROCESS - START");
// VARIABLES FROM XML
String folderId = null;
String srcDir = null;
String archDir = null;
Integer retentionPeriod;
String tarFileName = null;
// READ XML CONFIG FILE & UNMARSHALL; OBTAIN LIST OF FOLDERS
JAXBContext jc = JAXBContext.newInstance(Folders.class);
Unmarshaller unmarshaller = jc.createUnmarshaller();
Folders config = (Folders) unmarshaller.unmarshal(new FileReader(
new File(configFile)));
List folders = config.getFolder();
// LOOP THRU THE FOLDER LIST
for (Folder folder : folders) {
folderId = folder.getId();
logger.info("PROCESSING - " + folderId);
srcDir = folder.getFolderName();
if (isStringEmpty(srcDir)) {
logger.error("SRC DIRECTORY NOT SPECIFIED FOR " + folderId);
continue;
}
retentionPeriod = folder.getRetentionPeriodInDays();
logger.debug("RETENTION PERIOD - " + retentionPeriod);
archDir = folder.getArchivalFolder();
// CHECK FOR DELETE / ARCHIVE
if (folder.getOperation().equals(OperationType.DELETE)) {
logger.debug("DELETE: CHECKING FOR OLD FILES IN DIR - "
+ srcDir);
try {
dataRetention(new File(srcDir), retentionPeriod);
} catch (IOException ioEx) {
logger.error("error deleting old files for " + folderId,
ioEx);
}
} else if (folder.getOperation().equals(OperationType.ARCHIVE)) {
/*
* COMPRESS OLD FILES TO *.TAR.GZ AND MOVE TO ARCHIVE FOLDER.
* ALSO DELETE THE FILES ADDED TO THE ARCHIVE.
*/
try {
logger.debug("ARCHIVE: CHECKING FOR OLD FILES IN DIR - "
+ srcDir);
// CREATE TAR FILE
tarFileName = archiveFiles(srcDir, archDir,
retentionPeriod, folderId);
if (isStringEmpty(tarFileName)) {
logger.warn("ARCHIVAL PROCESS FAILED FOR " + folderId);
} else {
// GZIP FILE
gzipCompressFile(tarFileName);
// DELETE TAR FILE AFTER CREATION OF GZIP
deleteFile(new File(tarFileName));
}
} catch (FileNotFoundException fileEx) {
logger.error("file not found " + tarFileName, fileEx);
} catch (ArchiveException archiveEx) {
logger.error("housekeeping failed for " + folderId,
archiveEx);
} catch (IOException ioEx) {
logger.error("housekeeping failed for " + folderId, ioEx);
}
} else {
// INVALID OPTION
logger.warn("INVALID OPERATION. SHOULD BE DELETE / ARCHIVE");
}
// CONTINUE WITH NEXT RULE IF ANY
}
logger.info("HOUSE KEEPING PROCESS - COMPLETED");
}
/**
* CRAETE A TAR FILE
*
* @param strSrcDir
* src dir ending with File.separator
* @param strArchDir
* target archive dir ending with File.separator. tar files will
* be created here.
* @param archivalPeriod
* @throws FileNotFoundException
* @throws ArchiveException
* @throws IOException
*/
public String archiveFiles(String strSrcDir, String strArchDir,
Integer archivalPeriod, String folderId)
throws FileNotFoundException, ArchiveException, IOException {
// VALIDATIONS!
if (archivalPeriod == null) {
logger.warn("ARCHIVAL PERIOD NOT SPECIFIED FOR " + folderId);
return null;
}
if (isStringEmpty(strSrcDir)) {
logger.error("SRC DIRECTORY IS MISSING FOR " + folderId);
return null;
}
if (isStringEmpty(strArchDir)) {
logger.error("ARCHIVE DIRECTORY NOT SPECFIED FOR " + folderId
+ ". USE INPUT DIR AS ARCHIVE DIR.");
// SO ARCHIVE IN THE SRC DIR
strArchDir = strSrcDir;
}
File srcDir = new File(strSrcDir);
if (!srcDir.exists()) {
logger.warn("SKIP ARCHIVAL... INVALID SOURCE DIRECTORY "
+ strSrcDir);
return null;
}
File archDir = new File(strArchDir);
if (!archDir.exists()) {
// AIX UNIX - FOLDER WILL GET CREATED AUTOMATICALLY. BUT
// IN WINDOWS - ERROR!
logger.error("ARCHIVE DIRECTORY DOES NOT EXIST " + archDir);
return null;
}
// ARCHIVE
pathToBeExcluded = srcDir.getCanonicalPath();
logger.debug("pathToBeExcluded " + pathToBeExcluded);
File[] dirList = null; // LIST OF FILES IN SRC DIR
if (srcDir.isDirectory()) {
logger.debug("LOOKING FOR FILES IN " + srcDir.getCanonicalPath());
dirList = srcDir.listFiles();
if (dirList.length == 0) {
logger.warn("NO FILES FOUND IN DIR - " + strSrcDir);
return null;
}
} else {
// JUST ADD THE SINGLE FILE TO TAR
dirList = new File[] { srcDir };
pathToBeExcluded = srcDir.getCanonicalPath().substring(0,
srcDir.getCanonicalPath().lastIndexOf(File.separator));
}
// TAR FILE NAME WILL BE CURRENT DATE & TIMESTAMP
String tarFileName = strArchDir + getCurrentDateTime("yyyyMMddhhmm")
+ ".tar";
logger.debug("creating tarFile.... " + tarFileName);
File tarFile = new File(tarFileName);
tarFileName = tarFile.getCanonicalPath();
OutputStream out = null;
TarArchiveOutputStream aos = null;
try {
out = new FileOutputStream(tarFile);
aos = (TarArchiveOutputStream) new ArchiveStreamFactory()
.createArchiveOutputStream(ArchiveStreamFactory.TAR, out);
// LOOP THRU LIST OF FILES IN SRC DIR
for (File file : dirList) {
logger.trace("FileName:" + file.getCanonicalPath());
if (file.isDirectory()) {
// CHK SUB FOLDERS FOR FILES & ADD TO TAR
tarDir(file, aos, archivalPeriod);
} else if (file.isFile()) {
// TODO CHEK FOR FILENAME PATTERN
if (isFileOld(file, archivalPeriod)) {
tarFile(file, aos);
// DELETE THE FILE WHICH HAS BEEN ADDED TO TAR
deleteFile(file);
}
}
}
logger.info("TAR FILE CREATED SUCCESSFULLY - " + tarFileName);
} finally {
this.finish(aos);
this.close(aos);
this.close(out);
}
return tarFileName;
}
/**
* DELETE FILES OLDER THAN SPECIFED RETENTION PERIOD
*
* @param deleteDir
* @param retentionPeriod
* @throws IOException
*/
private void dataRetention(File deleteDir, Integer retentionPeriod)
throws IOException {
if (!deleteDir.exists()) {
logger.warn("SKIP RETENTION... INVALID DIRECTORY TO DELETE FILE FROM -"
+ deleteDir);
return;
}
if (retentionPeriod == null) {
logger.warn("SKIP RETENTION... RETENTION PERIOD NOT SPECIFIED");
return;
}
logger.info("CHECKING FOR OLD FILES IN DIR - "
+ deleteDir.getCanonicalPath());
File[] dirList = deleteDir.listFiles();
if (dirList.length == 0) {
logger.info("NO FILES FOUND IN " + deleteDir.getCanonicalPath());
return;
}
for (File file : dirList) {
logger.trace("processing file:" + file);
if (file.isDirectory()) {
// DO NOT DELETE DIRECTORIES!!!!
// RECURSIVELY CHECK SUB-DIRECTORIES FOR OLDER FILES
dataRetention(file, retentionPeriod);
} else if (file.isFile()) {
// TODO CHEK FOR FILENAME PATTERN
if (isFileOld(file, retentionPeriod)) {
deleteFile(file);
}
}
}
}
/**
*
* @param file
* file to be verified
* @param retentionPeriod
* number of days to retain files
* @return
* @throws IOException
*/
private boolean isFileOld(File file, int retentionPeriod)
throws IOException {
Date lastModifiedDate;
lastModifiedDate = new Date(file.lastModified());
logger.trace(file + " last modified on " + lastModifiedDate);
int diffInDays = getDifferenceInDays(currDate, lastModifiedDate);
logger.trace("DIFF IN DAYS:" + diffInDays);
if (diffInDays > retentionPeriod) {
logger.trace(file.getCanonicalPath() + " IS OLD. DAYS-"
+ diffInDays);
return true;
}
return false;
}
/**
* ADD FILE TO TAR ARCHIVE
*
* @param reqFile
* file to be verified
* @param aos
* @param retentionPeriod
* number of days to retain files
* @throws IOException
*/
private void tarDir(File reqFile, TarArchiveOutputStream aos,
int retentionPeriod) throws IOException {
logger.debug("looking for files in directory "
+ reqFile.getCanonicalPath());
File[] dirList = reqFile.listFiles();
// TODO - FIX EMPTY DIR CREATION IN TAR (NO ELIGIBLE FILE IS PRESENT).
// ADD A EMPTY DIRECTORY
addDirectoryToTar(reqFile, aos);
for (File file : dirList) {
logger.trace("processing file:" + file);
if (file.isDirectory()) {
// RECURSIVELY CHECK FOR FILES IN DIR
tarDir(file, aos, retentionPeriod);
} else if (file.isFile()) {
if (isFileOld(file, retentionPeriod)) {
// ADD FILE TO TAR
tarFile(file, aos);
// DELETE THE FILE WHICH HAS BEEN ADDED TO TAR
deleteFile(file);
}
}
}
}
/**
*
* @param file
* @param aos
* @throws IOException
*/
private void addDirectoryToTar(File file, TarArchiveOutputStream aos)
throws IOException {
try {
String dirName = getFileName(file);
logger.debug("ADD DIR TO TAR - " + dirName);
TarArchiveEntry entry = new TarArchiveEntry(file, dirName);
aos.putArchiveEntry(entry);
aos.closeArchiveEntry();
} catch (IOException ioEx) {
logger.warn("error adding dir to tar -" + file.getCanonicalPath(),
ioEx);
throw ioEx;
}
}
/**
*
* @param file
* @param aos
* @throws IOException
*/
private void tarFile(File file, TarArchiveOutputStream aos)
throws IOException {
TarArchiveEntry entry;
FileInputStream fin = null;
try {
String fileName = getFileName(file);
logger.debug("ADD FILE TO TAR - " + fileName);
entry = new TarArchiveEntry(file, fileName);
entry.setSize(file.length());
aos.putArchiveEntry(entry);
fin = new FileInputStream(file);
IOUtils.copy(fin, aos);
aos.closeArchiveEntry();
} catch (IOException ioEx) {
logger.warn("error adding file to tar -" + file.getCanonicalPath(),
ioEx);
throw ioEx;
} finally {
this.close(fin);
}
}
/**
*
* @param srcFileName
* file to gzipped!
* @throws FileNotFoundException
* @throws IOException
*/
private void gzipCompressFile(String srcFileName)
throws FileNotFoundException, IOException {
OutputStream out = null;
InputStream fin = null;
GzipCompressorOutputStream gzip = null;
String gzipFileName = srcFileName + ".gz";
logger.debug("CREATING GZIP FILE..... " + gzipFileName);
try {
File srcFile = new File(srcFileName);
if (!srcFile.exists() || !srcFile.isFile()) {
logger.error("INVALID FILE - " + srcFileName);
return;
}
fin = new FileInputStream(srcFile);
out = new FileOutputStream(gzipFileName);
gzip = new GzipCompressorOutputStream(out);
IOUtils.copy(fin, gzip);
logger.info("SUCCESSFULLY CREATED GZIP FILE - " + gzipFileName);
} finally {
this.close(fin);
this.close(gzip);
this.close(out);
}
}
/**
* DELETE SPECIFIED FILE
*
* @param file
* file to be deleted
* @throws IOException
*/
public void deleteFile(File file) throws IOException {
String strCanonicalPath = file.getCanonicalPath();
logger.debug("DELETING FILE " + strCanonicalPath);
// DO NOT DELETE ANY DIRECTORIES
if (!file.exists() || !file.isFile()) {
logger.error("INVALID FILE " + strCanonicalPath);
return;
}
boolean deleteStatus = file.delete();
if (!deleteStatus) {
logger.error("FAILED TO DELETE FILE " + strCanonicalPath);
}
}
/**
* derive file name to be used in tar. basically removes the directory path.
*
* @param file
* @return file name to be used in tar
* @throws IOException
*/
private String getFileName(File file) throws IOException {
// filename to be used in tar
String strCanonicalPath = file.getCanonicalPath();
String fileName = "";
// INDEX OF SRC
int index = strCanonicalPath.indexOf(pathToBeExcluded);
if (index >= 0) {
fileName = strCanonicalPath.substring(index
+ pathToBeExcluded.length() + 1);
}
logger.trace("Name to be used in TAR-" + fileName);
return fileName;
}
// ************ COMMON UTILITY FUNCTIONS ****************
/**
* CLOSE STREAM
*
* @param stream
* @throws IOException
*/
private void close(Closeable stream) throws IOException {
try {
if (stream != null) {
stream.close();
}
} catch (IOException ioEx) {
logger.warn("error closing stream", ioEx);
throw ioEx;
}
}
/**
* Ends the TAR archive without closing the underlying OutputStream
*
* @param aos
* @throws IOException
* on error
*/
private void finish(TarArchiveOutputStream aos) throws IOException {
try {
if (aos != null) {
aos.finish();
}
} catch (IOException ioEx) {
logger.warn("error while finish TarArchiveOutputStream", ioEx);
throw ioEx;
}
}
/**
* This method checks whether the string is empty or not
*
* @param strToCheck
* the string to be checked
* @return it returns true if the string is null or empty, otherwise it
* returns false
*/
public static boolean isStringEmpty(String strToCheck) {
if ((null == strToCheck) || strToCheck.trim().isEmpty()) {
return true;
} else {
return false;
}
}
/**
* get the difference in days between two dates. considers DST.
*
* @param date1
* the first date
* @param date1
* the second date
*
* @return int the difference in days (-ve if date1 < date2)
* @throws ParseException
*/
public static int getDifferenceInDays(Date date1, Date date2) {
Calendar cal1 = Calendar.getInstance();
cal1.setTime(date1);
Calendar cal2 = Calendar.getInstance();
cal2.setTime(date2);
// HAVE TO CONVERT BOTH THE INPUT DATES TO GMT TIMEZONE
// cal.getTimeZone().getOffset(cal.getTimeInMillis()) -> amount of time
// in milliseconds to add to GMT to get local time
long timeDiff = (cal1.getTimeInMillis() + cal1.getTimeZone().getOffset(
cal1.getTimeInMillis()))
- (cal2.getTimeInMillis() + cal2.getTimeZone().getOffset(
cal2.getTimeInMillis()));
return (int) TimeUnit.MILLISECONDS.toDays(timeDiff);
// long timeDiff = Math.abs(date1.getTime() - date2.getTime());
// return (int) TimeUnit.MILLISECONDS.toDays(timeDiff);
}
/**
* get current date
*
* @param dateFormat
* @return
*/
public static String getCurrentDateTime(String dateFormat) {
Date today = new Date();
DateFormat formatter = new SimpleDateFormat(dateFormat);
return formatter.format(today);
}
public static void main(String[] args) {
HouseKeeping tester = new HouseKeeping();
try {
tester.houseKeepingService();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}