import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashSet; import java.util.Iterator; import java.util.Vector; import oracle.jdbc.OracleCallableStatement; import oracle.jdbc.OraclePreparedStatement; class CodebundleAnalyser { static final SimpleDateFormat sdfTimestamp = new SimpleDateFormat( "dd.MM.yyyy HH:mm:ss"); static class ColumnData { String type=""; String key1=""; String key2=""; String key3=""; public boolean equals( Object o) { ColumnData c = (ColumnData) o; return type.equals( c.type) && key1.equals( c.key1) && key2.equals( c.key2) && key3.equals( c.key3); } public int hashCode() { int h; int l; l = key1.length(); h = l>0 ? (l>1 ? key1.charAt(l-1)+key1.charAt(l-2)<<8 : key1.charAt(l-1)) : 0; l = key2.length(); h = h<<8 | (l>0 ? (l>1 ? key2.charAt(l-1)+key2.charAt(l-2)<<8 : key2.charAt(l-1)) : 0); l = key3.length(); h = h<<8 | (l>0 ? (l>1 ? key3.charAt(l-1)+key3.charAt(l-2)<<8 : key3.charAt(l-1)) : 0); l = type.length(); h = h<<8 | (l>0 ? (l>1 ? type.charAt(l-1)+type.charAt(l-2)<<8 : type.charAt(l-1)) : 0); return h; } } static class ColumnGroup { String type; Vector vColumns = new Vector(); HashSet setUniqueData = new HashSet(); } static String newLine = System.getProperty("line.separator"); static String url = "jdbc:oracle:thin:codebundles/codebundles@" + "(DESCRIPTION = " + "(ADDRESS = " + "(COMMUNITY = TCP.DE.IKEA.COM)" + "(PROTOCOL = TCP)" + "(Host = CNSBAC.SEUROPE.IKEA.COM)" + "(Port = 1526) )" + "(CONNECT_DATA = " + "(SID = CNSDEV)" + "(GLOBAL_NAME = CNSDEV.TEST.IKEA.COM) ) )"; String baseDirectory; // = "CNS$APPL:[USER.BSWG.DEV.CODEBUNDLES]"; String unpackDirectory; // = "CNS$APPL:[USER.BSWG.DEV.CODEBUNDLES.UNPACK]"; String[] a_excludeAgreement; String backupFile; SimpleDateFormat CBDateFormat = new SimpleDateFormat("dd.MM.yy HH:mm:ss"); BufferedReader in = null; int nLine = 0; boolean bOneCodebundle = false; //Argument is a codebundle VMSTools vmsTools; DBLogger logger; Connection conn; IniReader iniReader; public CodebundleAnalyser( String args[], Connection connIn, IniReader iniReaderIn, DBLogger loggerIn) { iniReader = iniReaderIn; conn = connIn; logger = loggerIn; } public static void usage() { System.err.println( "Usage: CodebundleAnalyser <backup file name> [options]"); System.err.println( ""); System.err.println( "Options: -r Resume interrupted import"); System.err.println( " -o <backup file name> is a codebundle. Import only this one. (Debugging)"); } public java.sql.Timestamp parseCodebundleDate( String s) { java.sql.Timestamp d = null; try { d = new java.sql.Timestamp( CBDateFormat.parse( s).getTime()); } catch (ParseException e) { System.err.println( "Cannot parse date: " + s); } return d; } /************************************************************************************************ *************************************************************************************************/ public String readLine() throws java.io.IOException { nLine++; String s = in.readLine(); //System.err.println( "Read:" + s); return s; } /************************************************************************************************ *************************************************************************************************/ public static String formatTimestamp( String sTimestamp) { return sTimestamp.substring( 6, 8) + "." + sTimestamp.substring( 4, 6) + "." + sTimestamp.substring( 0, 4) + " " + sTimestamp.substring( 8, 10) + ":" + sTimestamp.substring( 10, 12) + ":" + sTimestamp.substring( 12, 14); } /************************************************************************************************ *************************************************************************************************/ public static java.sql.Timestamp getDatetime( String s) { return( new java.sql.Timestamp( new GregorianCalendar( 1900 + Integer.parseInt( s.substring( 6, 10)), Integer.parseInt( s.substring( 3, 5)), Integer.parseInt( s.substring( 0, 2)), Integer.parseInt( s.substring( 11, 13)), Integer.parseInt( s.substring( 14, 16)), Integer.parseInt( s.substring( 17, 19)) ).getTime().getTime() )); } /************************************************************************************************ *************************************************************************************************/ public Vector filterColumns( Vector vColumnsIn, String sBackupFilename, String sFilename) { Vector vGroups = new Vector(); boolean csm_no_found = false; boolean csm_found = false; boolean shp_no_found = false; boolean shp_found = false; for (int i=0; i<vColumnsIn.size() - 2; i++) { CodebundleColumn col1 = (CodebundleColumn) vColumnsIn.elementAt( i); CodebundleColumn col2 = (CodebundleColumn) vColumnsIn.elementAt( i+1); CodebundleColumn col3 = (CodebundleColumn) vColumnsIn.elementAt( i+2); //Scan for consignment identifier if( col3.name.equals( "CSM_NO") &&(col1.name.equals( "BU_CODE_CRE") || col1.name.equals( "BU_CODE_CRE_CSM")) &&(col2.name.equals( "BU_TYPE_CRE") || col2.name.equals( "BU_TYPE_CRE_CSM"))) { ColumnGroup cg = new ColumnGroup(); cg.type = "CSM"; cg.vColumns.add( col1); cg.vColumns.add( col2); cg.vColumns.add( col3); vGroups.add( cg); csm_found = true; } else if( col1.name.equals( "CSM_NO") //FREJA+DISCUS style &&(col2.name.equals( "BU_CODE_CRE") || col2.name.equals( "BU_CODE_CRE_CSM")) &&(col3.name.equals( "BU_TYPE_CRE") || col3.name.equals( "BU_TYPE_CRE_CSM"))) { ColumnGroup cg = new ColumnGroup(); cg.type = "CSM"; cg.vColumns.add( col2); cg.vColumns.add( col3); cg.vColumns.add( col1); vGroups.add( cg); csm_found = true; } if( col3.name.equals( "CSM_NO")) { csm_no_found = true; } //Scan for shipment identifier if( col3.name.equals( "SHP_NO") &&(col1.name.equals( "BU_CODE_CRE") || col1.name.equals( "BU_CODE_CRE_SHP")) &&(col2.name.equals( "BU_TYPE_CRE") || col2.name.equals( "BU_TYPE_CRE_SHP"))) { ColumnGroup cg = new ColumnGroup(); cg.type = "SHP"; cg.vColumns.add( col1); cg.vColumns.add( col2); cg.vColumns.add( col3); vGroups.add( cg); shp_found = true; } else if( col1.name.equals( "SHP_NO") //FREJA+DISCUS style &&(col2.name.equals( "BU_CODE_CRE") || col2.name.equals( "BU_CODE_CRE_SHP")) &&(col3.name.equals( "BU_TYPE_CRE") || col3.name.equals( "BU_TYPE_CRE_SHP"))) { ColumnGroup cg = new ColumnGroup(); cg.type = "SHP"; cg.vColumns.add( col2); cg.vColumns.add( col3); cg.vColumns.add( col1); vGroups.add( cg); shp_found = true; } if( col3.name.equals( "SHP_NO")) { shp_no_found = true; } } if( csm_no_found && !csm_found) { logger.log( conn, "CodebundleAnalyser.filterColumns " + sBackupFilename + "." + sFilename + " :no complete consignment identifier found"); } if( shp_no_found && !shp_found) { logger.log( conn, "CodebundleAnalyser.filterColumns " + sBackupFilename + "." + sFilename + " :no complete shipment identifier found"); } return vGroups; } /************************************************************************************************ *************************************************************************************************/ public void analyse( String sBackupFilename, String sPath, String sFilename, VMSFileInfo viCB) throws IOException, SQLException { String line; String sITable; String sTarget; String sSender; String sAgreement; String sSendDate = ""; PreparedStatement psObject = null; java.sql.Timestamp fileDate; nLine = 0; try { in = new BufferedReader( new FileReader( sPath + sFilename)); line = readLine(); //Ignore 3 lines line = readLine(); //Exclude uninteresting agreements sAgreement = line.substring( 22, 30); for( int i=0; i<a_excludeAgreement.length; i++) { if( 0 <= sAgreement.indexOf( a_excludeAgreement[i])) { logger.log( conn, "Excluded because of INI-file setting ExcludeAgreement:" + sAgreement + " in " + sBackupFilename + "/" + sFilename); in.close(); return; //>>>>>>>>>>>>>>>>>>>>>>>> } } sTarget = line.substring( 112).trim(); line = readLine(); sSendDate = formatTimestamp( line.substring( 20, 34)); boolean bCBInserted = false; while( in.ready()) { Vector vTempColumns = new Vector(); line = readLine(); sITable = line.substring( 3, 12).trim(); sSender = line.substring( 51, 59).trim(); fileDate = VMSTools.parseVMSFileDate( viCB.fileDate); if (!bCBInserted) { PreparedStatement psInsCB = null; try { String sSQL = "insert into codebundle_import( backup_file_name, file_name, file_date, " + "file_size, sender, target, send_time) values(?,?,?,?,?,?,?)"; psInsCB = conn.prepareStatement( sSQL); ((OraclePreparedStatement)psInsCB).setExecuteBatch( 1); // Create a Statement psInsCB.setString( 1, sBackupFilename); psInsCB.setString( 2, sFilename); psInsCB.setTimestamp( 3, fileDate); psInsCB.setInt( 4, viCB.fileSize); psInsCB.setString( 5, sSender); psInsCB.setString( 6, sTarget); psInsCB.setTimestamp( 7, parseCodebundleDate( sSendDate)); psInsCB.executeUpdate(); bCBInserted = true; } catch (SQLException e2) { if( e2.getErrorCode() == 1654) //Tablespace problem { throw e2; //Do not continue>>>>>>>>>>>>>>>>> } logger.log( conn, "CodebundleAnalyser.analyse: Error: " + e2.getMessage().trim()); logger.log( conn, " Tried to insert into codebundle_import: " + sBackupFilename + "," + viCB.fileName + "," + sTarget + "," + sSendDate); in.close(); return; //>>>>>>>>>>>>>>>>> } finally { try { if( psInsCB != null) psInsCB.close(); } catch (SQLException e3){} } } // String sSql = "INSERT INTO cns_object_import (backup_file_name, file_name, " + String sSql = "INSERT INTO cns_object_t (backup_file_name, file_name, " + "itable, obj_type, key1, key2, key3, file_month) " + "VALUES(?,?,?,?,?,?,?,?)"; try { psObject = conn.prepareStatement( sSql); ((OraclePreparedStatement)psObject).setExecuteBatch( 50); } catch (Exception e) { e.printStackTrace(); logger.log( conn, "CodebundleAnalyser.analyse: Cannot prepare statement:" + sSql + newLine); logger.log( conn, e.getMessage() + newLine); return; //>>>>>>>>>>>>>>>>>> } // //Read Data Definition // while( in.ready()) { line = readLine(); if (line.length() < 1) { logger.log( conn, "CodebundleAnalyser.analyse: Unexpected empty line: " + nLine); return; //Jump to finally block >>>>>>>>>>>>>>>>>> } if (line.charAt(0) != 'C') break; //Continue with reading data >>>>>>>>>>>>>>>> CodebundleColumn col = new CodebundleColumn(); int res = 0; try { res = col.readType( line); } catch (RuntimeException e) { e.printStackTrace(); logger.log( conn, "CodebundleAnalyser.analyse: Unexpected incomplete line: " + line); return; //Jump to finally block >>>>>>>>>>>>>>>>>> } if ( 0 != res) { logger.log( conn, "CodebundleAnalyser.analyse: Unexpected data type " + res + " in line " + nLine + " " + sBackupFilename + " " + sFilename); return; //Jump to finally block >>>>>>>>>>>>>>>>>> } vTempColumns.addElement( col); if( vTempColumns.size() > 1) { ((CodebundleColumn) vTempColumns.elementAt( vTempColumns.size() - 2)).endindex = col.startindex - 1; } } Vector vGroups = filterColumns( vTempColumns, sBackupFilename, sFilename); // //Read Data // int nLine = 1; int numRows = 0; boolean continueScan = true; //Stop if no interesting columns found // do { if( continueScan && line.length() > 1) { if (line.charAt(0) != 'D') break; // New I-table or EOF >>>>>>>>>>>>>>>> for (int g=0; g<vGroups.size(); g++) //Loop through all column groups (CSM, SHP) { try { ColumnGroup cg = (ColumnGroup) vGroups.elementAt( g); for (int i=0; i<cg.vColumns.size(); i++) //Loop through the keys of one group { ((CodebundleColumn) cg.vColumns.elementAt( i)).readData( line); } ColumnData cd = new ColumnData(); cd.type = cg.type; cd.key1 = ((CodebundleColumn) cg.vColumns.elementAt( 0)).undecoratedValue; cd.key2 = ((CodebundleColumn) cg.vColumns.elementAt( 1)).undecoratedValue; cd.key3 = ((CodebundleColumn) cg.vColumns.elementAt( 2)).undecoratedValue; if (cd.key1.length() > 0 && cd.key2.length() > 0 && cd.key3.length() > 0) { //This must be changed together with the primary key of the table if we //get Objects with a number of key-parts <> 3 cg.setUniqueData.add( cd); } } catch( java.lang.StringIndexOutOfBoundsException e) { logger.log( conn, "CodebundleAnalyser.analyse: Error:" + sBackupFilename + " " + sFilename + ": " + e.getMessage()); } } numRows++; /* if (numRows % 1000 == 0) { System.err.println( "Analyser:" + numRows); } */ } if (!in.ready()) break; //>>>>>>>>>>>>>>>>>> line = readLine(); nLine++; } while( true); ColumnGroup cg = null; ColumnData cd = null; try { //System.out.println( "Groups: " + vGroups.size()); for (int g=0; g<vGroups.size(); g++) //Loop through all column groups (CSM, SHP) { cg = (ColumnGroup) vGroups.elementAt( g); psObject.setString( 1, sBackupFilename); // backupfile psObject.setString( 2, sFilename); // bundlename psObject.setString( 3, sITable); // I-Table psObject.setString( 4, cg.type ); // type psObject.setInt( 8, fileDate.getMonth() + 1); Iterator it = cg.setUniqueData.iterator(); while( it.hasNext()) { cd = (ColumnData) it.next(); psObject.setString( 5, cd.key1); psObject.setString( 6, cd.key2); psObject.setString( 7, cd.key3); psObject.executeUpdate(); /* System.out.println( "Insert: " + sBackupFilename + "," + sFilename + "," + sITable + "," + cg.type + "," + cd.key1 + "," + cd.key2 + "," + cd.key3); */ } } } catch (SQLException e1) { String s = e1.getMessage(); logger.log( conn, "CodebundleAnalyser.analyse: Error: " + e1.getMessage().trim()); logger.log( conn, " Tried to insert: " + sBackupFilename + "," + sFilename + "," + sITable + "," + cg.type + "," + cd.key1 + "," + cd.key2 + "," + cd.key3); if( s.indexOf( "ORA-02291") != -1) { e1.printStackTrace(); return; //>>>>>>>>>>>>>>>>>>>>>>>> } } finally { try { if( psObject != null) psObject.close(); } catch (SQLException e2){} } } } finally { if( in != null) in.close(); } } /************************************************************************************************ *************************************************************************************************/ public void readInifile( IniReader iniReader) { url = iniReader.getPropertyString( "Database", "ConnectString", url); baseDirectory = iniReader.getPropertyString( "Settings", "BaseDirectory", baseDirectory); unpackDirectory = iniReader.getPropertyString( "Settings", "UnpackDirectory", unpackDirectory); String excludeAgreement = iniReader.getPropertyString( "Settings", "ExcludeAgreement", ""); a_excludeAgreement = VMSTools.split(excludeAgreement, ','); } /************************************************************************************************ *************************************************************************************************/ public boolean readCommandline( String args[]) { int argsUsed = 0; for (int i = 0; i < args.length; i++) { if (args[i].charAt(0) == '-') { switch( args[i].toLowerCase().charAt(1)) { case 'h': case '?': usage(); return false; case 'o': bOneCodebundle = true; break; default: System.err.println( "*** Unknown option:" + args[argsUsed]); usage(); break; } } else { if (argsUsed==0) backupFile = args[i].toUpperCase(); argsUsed++; } } // if (argsUsed < 1) { usage(); return false; } return true; } /************************************************************************************************ *************************************************************************************************/ public void updateStatus( String file_name, int files_done, int files_todo) throws SQLException { String s = "begin import_status.upd_cb_status(?,?,?); end;"; OracleCallableStatement cs = (OracleCallableStatement) conn.prepareCall( s ); cs.setString( 1, file_name); cs.setInt( 2, files_done); cs.setInt( 3, files_todo); cs.execute(); cs.close(); /* PreparedStatement psInsStatus = null; try { String s = "update status_t set backup_file_name=?," + (startedEnded != null ? startedEnded + "," : "") + "files_done=?,files_todo=? "; psInsStatus = conn.prepareStatement( s); psInsStatus.setString( 1, backup_file_name); psInsStatus.setInt( 2, files_done); psInsStatus.setInt( 3, files_todo); psInsStatus.executeUpdate(); psInsStatus.close(); } catch (SQLException e) { logger.log( conn, "CodebundleAnalyser.updateStatus " + e.getMessage()); } finally { try { if( psInsStatus != null) psInsStatus.close(); } catch (SQLException e2){} } */ } /************************************************************************************************ *************************************************************************************************/ public void readCodebundles( String directory, VMSFileInfo viBackup, boolean bResume) throws SQLException, IOException { if( vmsTools == null) { vmsTools = new VMSTools( baseDirectory, "cbaListing.tmp", "cmdCBA.com"); } vmsTools.purgeCmdFile(); try { Vector vCBs = vmsTools.dir( directory); PreparedStatement psInsBF = null; if( !bResume) { try { psInsBF = conn.prepareStatement( "insert into backup_file_import(backup_file_name, file_size) values(?,?)"); psInsBF.setString( 1, viBackup.fileName); psInsBF.setInt( 2, viBackup.fileSize); psInsBF.executeUpdate(); psInsBF.close(); } catch (SQLException e) { logger.log( conn, "CodebundleAnalyser.readCodebundles: " + e.getMessage().trim()); logger.log( conn, " Tried to insert into backup_file_import: " + viBackup.fileName); psInsBF.close(); } } String bundleName; for( int i=0; i<vCBs.size(); i++) { VMSFileInfo vi = (VMSFileInfo) vCBs.elementAt( i); bundleName = vi.fileName.toUpperCase(); updateStatus( bundleName, i, vCBs.size()); java.util.Date dStart = new java.util.Date(); // analyse( viBackup.fileName, directory, bundleName, vi); // java.util.Date d = new java.util.Date(); System.err.println( sdfTimestamp.format( new Date()) + ": " + "Codebundle " + i + ": " + bundleName + ": " + (d.getTime() - dStart.getTime()) + " ms" + " " + ((i+1) * 100)/vCBs.size() + "% done"); conn.commit(); //Commit every codebundle new File( directory + bundleName).delete(); } updateStatus( null, vCBs.size(), 0); } catch (IOException e1) { e1.printStackTrace(); throw e1; } } /************************************************************************************************ *************************************************************************************************/ public void run() throws IOException { try { logger.initLog( conn); VMSFileInfo vi = new VMSFileInfo( backupFile, "01-JAN-00", 0 ); if (bOneCodebundle) //Just for testing { logger.log( conn, "CodebundleAnalyser started for dummy analysis"); try { analyse( "dummy", unpackDirectory, backupFile, vi); } catch (IOException e2) { e2.printStackTrace(); } return; //>>>>>>>>>>>>>>>>>> } vi.fileName = backupFile; logger.log( conn, "CodebundleAnalyser started"); readCodebundles( unpackDirectory, vi, false); logger.log( conn, "CodebundleAnalyser ended"); conn.commit(); logger.exitLog(); conn.close(); } catch (SQLException e) { logger.log( conn, "CodebundleAnalyser.run: " + e.getMessage().trim()); e.printStackTrace(); } } /************************************************************************************************ *************************************************************************************************/ private boolean initializeStandalone( String args[]) { if (!readCommandline( args)) { return false; } if( iniReader == null) { try { iniReader = new IniReader( "CodebundleAnalyser.ini"); } catch (FileNotFoundException e1) { System.err.println( e1.getMessage()); e1.printStackTrace(); return false; //>>>>>>>>>>>>>>> } catch (IOException e1) { System.err.println( e1.getMessage()); e1.printStackTrace(); return false; //>>>>>>>>>>>>>>> } } //INI file readInifile( iniReader); System.err.println( "Using base directory: " + baseDirectory); System.err.println( "Using source directory: " + unpackDirectory); if (conn == null) { conn = connect(); } return conn != null; } /************************************************************************************************ * Connect to database in standalone mode *************************************************************************************************/ private static Connection connect() { Connection conn = null; try { // Load the Oracle JDBC driver DriverManager.registerDriver( new oracle.jdbc.OracleDriver()); System.err.println( "Connecting to " + url + ": "); conn = DriverManager.getConnection( url); System.err.println( " Success"); System.err.println(); // conn.setAutoCommit( false); } catch (SQLException e) { System.err.println( "Connection failed:" + e.getMessage().trim()); e.printStackTrace(); } return conn; } /************************************************************************************************ *************************************************************************************************/ public static void main( String args []) { DBLogger logger = new DBLogger(); CodebundleAnalyser thisAnalyser = new CodebundleAnalyser( args, null, null, logger); //initializeStandalone must be called outside the constructor //because it is not needed when we use CodebundleAnalyser in //another object if( thisAnalyser.initializeStandalone( args)) { // if (false) { try { thisAnalyser.run(); thisAnalyser.conn.commit(); thisAnalyser.conn.close(); } catch (SQLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } } } }