Upload Multiple CSV File Parallel | Java Example

This application can be used to upload mulitple csv files parallely using edbloader and insert data in table of PostgreSQL.

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;


public class ImplementPlLoader {

 private String ImplementPlLoader1(String csvFile, String badFile,String ctlFile, String parFile, String logFile,String edbLoaderpath,String tableName, int num) throws Exception 
 {
  String[] edbLoaderpathStr = edbLoaderpath.split(",");
  String[] csvFileStr = csvFile.split(",");
  String[] badFileStr = badFile.split(",");
  String[] ctlFileStr = ctlFile.split(",");  
  String[] parFileStr = parFile.split(",");
  String[] logFileStr = logFile.split(",");
  String parFileData;
  String ctlData;
  boolean success = false;
  String[] cmd = new String[2];
  String exitValue = "";
  
  for(int i=0; i<num; i++)
  {
   System.out.println("edbLoaderpathStr["+i+"]-"+edbLoaderpathStr[i]);
   if(!(new File(edbLoaderpathStr[i])).exists())
    (new File(edbLoaderpathStr[i])).mkdirs();
   
   System.out.println("csvFileStr[i].... "+csvFileStr[i]);
   System.out.println("badFileStr[i].... "+badFileStr[i]);
   System.out.println("ctlFileStr[i] "+ctlFileStr[i]);
   System.out.println("parFileStr[i] "+parFileStr[i]);
   System.out.println("logFileStr[i] "+logFileStr[i]);
   
   if (!(new File(edbLoaderpathStr[i]+csvFileStr[i])).exists())
    return "Fail1";
   
   parFileData = "CONTROL="+ctlFileStr[i]+" BAD="+badFileStr[i]+" LOG="+logFileStr[i];
   System.out.println("parFileData: "+parFileData);

   ctlData = "LOAD DATA INFILE '"+edbLoaderpathStr[i]+csvFileStr[i]+"' INTO TABLE "+tableName+" APPEND FIELDS TERMINATED BY ','  OPTIONALLY ENCLOSED BY '"' TRAILING NULLCOLS (empid,empname)";
   System.out.println("ctlData :: " + ctlData);
   
   if ((new File(edbLoaderpathStr[i]+ctlFileStr[i])).exists())
    success=(new File(edbLoaderpathStr[i]+ctlFileStr[i])).delete();
   else
    new File(edbLoaderpathStr[i]+ctlFileStr[i]).createNewFile();

   BufferedWriter out = new BufferedWriter(new FileWriter(edbLoaderpathStr[i]+ctlFileStr[i]));
   out.write(ctlData);
   out.close();

   System.out.println("ctl file success :::" +success);

   /** Creating para file */
   if ((new File(edbLoaderpathStr[i]+parFileStr[i])).exists())
    success=(new File(edbLoaderpathStr[i]+parFileStr[i])).delete();
   else
    new File(edbLoaderpathStr[i]+parFileStr[i]).createNewFile();

   BufferedWriter outParFile = new BufferedWriter(new FileWriter(edbLoaderpathStr[i]+parFileStr[i]));
   outParFile.write(parFileData);
   outParFile.close();

   System.out.println("Para file success :::" +success);
   
   cmd[0]="edbldr";
   cmd[1]=" -d SAM -p 5444 -h localhost USERID=enterprisedb/Admin@123 DIRECT=TRUE PARALLEL=TRUE PARFILE="+parFileStr[i];
   System.out.println("cmd[0] "+cmd[0]);

   String[] env=null;
   Process p = null;

   int iReturn=1;
   long start=System.currentTimeMillis();
   long end=System.currentTimeMillis();

   System.out.println("start:"+start);
   
   while(iReturn!=0 && (end-start)/1000<=5)
   {
    Runtime rt = Runtime.getRuntime();
    p = rt.exec(cmd,env,new File(edbLoaderpathStr[i]));
    iReturn = p.waitFor();
    end=System.currentTimeMillis();
    System.out.println("end:"+end);
    System.out.println("iReturn:"+iReturn);
    System.out.println("exitVal:"+p.exitValue());

    if(p.exitValue()==2)
    {
     exitValue= "2";   // row discarded
     break;
    }
   }
   if((new File(badFile)).exists())
    return "-1";

   if(exitValue.equals("2"))  // row discarded
    return "-2";
  }
  
  return "1";
 }
 
 public static void main(String[] args) throws Exception {
  ImplementPlLoader implPlLoader =  new ImplementPlLoader();
  
  String csvFile = "empFileCsv1.csv,empFileCsv2.csv"; // Name of Csv Files
  String badFile = "empFileBad1.bad,empFileBad2.bad"; 
  String ctlFile = "empFileCtl1.ctl,empFileCtl2.ctl";
  String parFile = "empFilePar1.par,empFilePar2.par";
  String logFile = "empFileLog1.log,empFileLog2.log";
  
  String edbLoaderpath = "D://Sam//edbldr1//,D://Sam//edbldr2//";
  String tableName = "EMP_PL";
  String str = implPlLoader.ImplementPlLoader1(csvFile,badFile,ctlFile,parFile,logFile,edbLoaderpath,tableName,2);
  System.out.println("return str "+str);
  
  if(str.equals("Fail1"))
   System.out.println("Data file doesnot exist");
 }

 

}

[ Read More ]