From 65865f60fd6cbd0f2cf82d4161ab21ccca660ee3 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Mon, 5 Apr 2021 23:56:13 -0700 Subject: [PATCH 1/3] Added Excel to CSV file converter --- dataspread-ui/src/Components/Menu/import.js | 4 +- webapp/pom.xml | 22 ++ webapp/src/api/XLStoCSV.java | 316 ++++++++++++++++++ .../src/api/controller/ImportController.java | 16 + 4 files changed, 356 insertions(+), 2 deletions(-) create mode 100644 webapp/src/api/XLStoCSV.java diff --git a/dataspread-ui/src/Components/Menu/import.js b/dataspread-ui/src/Components/Menu/import.js index 2fdb51175..e4eaeb4da 100755 --- a/dataspread-ui/src/Components/Menu/import.js +++ b/dataspread-ui/src/Components/Menu/import.js @@ -81,9 +81,9 @@ export default class ModalImportFile extends Component { org.springframework spring-websocket + + + + org.apache.poi + poi + 5.0.0 + + + + + org.apache.poi + poi-ooxml + 5.0.0 + + + + org.apache.xmlbeans + xmlbeans + 4.0.0 + + + DataSpread diff --git a/webapp/src/api/XLStoCSV.java b/webapp/src/api/XLStoCSV.java new file mode 100644 index 000000000..8d08d1ef6 --- /dev/null +++ b/webapp/src/api/XLStoCSV.java @@ -0,0 +1,316 @@ +package api; +import java.util.*; +import com.opencsv.CSVWriter; +import org.apache.poi.hssf.usermodel.HSSFEvaluationWorkbook; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.formula.*; +import org.apache.poi.ss.formula.ptg.*; +import org.apache.poi.xssf.usermodel.XSSFEvaluationWorkbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.ss.usermodel.*; +import java.io.*; + +public class XLStoCSV { + private final Map sheetOffsets; + private final Workbook workbook; + private final FormulaParsingWorkbook parsingWorkbook; + public XLStoCSV(File excelFile) throws IOException { + workbook = WorkbookFactory.create(excelFile); + String fileExt = getFileExt(excelFile.getName()); + if (fileExt.equals("xls")) { + parsingWorkbook = HSSFEvaluationWorkbook.create((HSSFWorkbook) workbook); + } else if (fileExt.equals("xlsx")){ + parsingWorkbook = XSSFEvaluationWorkbook.create((XSSFWorkbook) workbook); + } else throw new IllegalArgumentException("File must have either \".xls\" or \".xlsx\" as its extension"); + sheetOffsets = new HashMap(); + } + + /** + * + * @param csv + * + * Converts excel file to csv file + */ + public void fullConvert(File csv) { + try { + CSVWriter out = new CSVWriter(new FileWriter(csv)); + assert workbook != null; + int longestRow = setUp(); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + Sheet sheet = workbook.getSheetAt(i); + for (int r = 0, rEnd = sheet.getLastRowNum(); r <= rEnd; r++) { + Row row = sheet.getRow(r); + String[] line = new String[longestRow + 1]; + if (row != null) { + for (int c = 0; c <= longestRow; c++) { + org.apache.poi.ss.usermodel.Cell cell = row.getCell(c, + Row.MissingCellPolicy.RETURN_BLANK_AS_NULL); + if (cell != null) { + line[c] = getCellTextWithOffset(cell); + } + } + } + + out.writeNext(line); + } + } + out.close(); + workbook.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * + * @param excel Excel file to be read from + * @param csv CSV file to be written to + * + * Converts Excel .xls or .xlsx file to a CSV file + */ + static public void basicFullConvert(File excel, File csv) { + try { + CSVWriter out = new CSVWriter(new FileWriter(csv)); + Workbook wb = WorkbookFactory.create(excel); + assert wb != null; + int longestRow = getLongestRowInWB(wb); + + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + //for (int i = 0; i < 1; i++) { // if converting all sheets, replace this line with line above + Sheet sheet = wb.getSheetAt(i); + for (int r = 0, rEnd = sheet.getLastRowNum(); r <= rEnd; r++) { + Row row = sheet.getRow(r); + String[] line = new String[longestRow + 1]; + if (row != null) { + for (int c = 0; c <= longestRow; c++) { + org.apache.poi.ss.usermodel.Cell cell = row.getCell(c, + Row.MissingCellPolicy.RETURN_BLANK_AS_NULL); + if (cell != null) { + line[c] = getCellText(cell); + } + } + } + out.writeNext(line); + } + } + out.close(); + wb.close(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * + * @return longest row in the Excel wb + * + * adds sheet number to row offset maps to sheetOffsets + */ + private int setUp() { + int longestRow = 0; + int offset = 0; + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + Sheet sheet = workbook.getSheetAt(i); + sheetOffsets.put(i, offset); + System.out.println("Added sheet " + i + ": " + "\t(Offset: " + offset + ")\t" + sheet.getSheetName()); + offset += sheet.getLastRowNum() + 1; + for (int r = 0, rEnd = sheet.getLastRowNum(); r <= rEnd; r++) { + Row row = sheet.getRow(r); + if (row != null) { + //offset++; + if (row.getLastCellNum() > longestRow) { + longestRow = row.getLastCellNum(); + } + } + } + } + return longestRow; + } + + /** + * + * @param cell + * @return the string representaion of the cell with translated cell coordinates + */ + private String getFormulaWithOffset(org.apache.poi.ss.usermodel.Cell cell) { + assert(cell.getCellType().equals(CellType.FORMULA)); + Ptg[] tokens = FormulaParser.parse( + cell.getCellFormula(), + parsingWorkbook, + FormulaType.CELL, + workbook.getSheetIndex(cell.getSheet()), + cell.getRowIndex()); + for (int i = 0; i < tokens.length; i++) { + Ptg token = tokens[i]; + if (token instanceof Area3DPtg) { + Area3DPtg token3D = (Area3DPtg) token; + // TODO: fix ptg sheet numbering + int sheetIndex = token3D.getExternSheetIndex(); + String refString = token3D.toFormulaString((FormulaRenderingWorkbook) parsingWorkbook); + String sheetName = refString.substring(0, refString.indexOf('!')); + int offset = sheetOffsets.get(workbook.getSheetIndex(sheetName)); + tokens[i] = new AreaPtg(token3D.getFirstRow() + offset, token3D.getLastRow() + offset, + token3D.getFirstColumn(), token3D.getLastColumn(), + token3D.isFirstRowRelative(), token3D.isLastRowRelative(), + token3D.isFirstColRelative(), token3D.isLastColRelative()); + } else if (token instanceof AreaPtg) { + AreaPtg tokenArea = (AreaPtg) token; + int offset = sheetOffsets.get(workbook.getSheetIndex(cell.getSheet().getSheetName())); + tokens[i] = new AreaPtg(tokenArea.getFirstRow() + offset, tokenArea.getLastRow() + offset, + tokenArea.getFirstColumn(), tokenArea.getLastColumn(), + tokenArea.isFirstRowRelative(), tokenArea.isLastRowRelative(), + tokenArea.isFirstColRelative(), tokenArea.isLastColRelative()); + } else if (token instanceof RefPtg) { + RefPtg tokenRef = (RefPtg) token; + int offset = sheetOffsets.get(workbook.getSheetIndex(cell.getSheet().getSheetName())); + tokens[i] = new RefPtg(tokenRef.getRow() + offset, tokenRef.getColumn(), + tokenRef.isRowRelative(), tokenRef.isColRelative()); + } else if (token instanceof Ref3DPtg) { + Ref3DPtg token3D = (Ref3DPtg) token; + // TODO: fix ptg sheet numbering + int sheetIndex = token3D.getExternSheetIndex(); + String refString = token3D.toFormulaString((FormulaRenderingWorkbook) parsingWorkbook); + String sheetName = refString.substring(0, refString.indexOf('!')); + int offset = sheetOffsets.get(workbook.getSheetIndex(sheetName)); + tokens[i] = new RefPtg(token3D.getRow() + offset, token3D.getColumn(), + token3D.isRowRelative(), token3D.isColRelative()); + } + } + return FormulaRenderer.toFormulaString((FormulaRenderingWorkbook) parsingWorkbook, tokens); + } + + /** + * + * @param wb + * @return returns length of longest row in the wb + */ + static private int getLongestRowInWB(Workbook wb) { + int longestRow = 0; + for (int i = 0; i < wb.getNumberOfSheets(); i++) { + Sheet sheet = wb.getSheetAt(i); + // Find longest row -- CSV reader requires all rows to have same number of delimiters + for (int r = 0, rEnd = sheet.getLastRowNum(); r <= rEnd; r++) { + Row row = sheet.getRow(r); + if (row != null && row.getLastCellNum() > longestRow) { + longestRow = row.getLastCellNum(); + } + } + } + return longestRow; + } + + /** + * + * @param fileName + * @return file extension of fileName + */ + private String getFileExt(String fileName) { + int i = fileName.lastIndexOf('.'); + if (i > 0) { + return fileName.substring(i + 1); + } else return ""; + } + + /** + * + * @param cell A cell in the Workbook + * @return A string representation of the cell. + */ + static private String getCellText(org.apache.poi.ss.usermodel.Cell cell) { + try { + String value; + org.apache.poi.ss.usermodel.CellType cellType = cell.getCellType(); + switch (cellType) { + case NUMERIC: + value = getNumericCellValue(cell); + break; + case STRING: + value = cell.getStringCellValue(); + break; + case FORMULA: + value = "=" + cell.getCellFormula(); + break; + case BOOLEAN: + value = cell.getBooleanCellValue() ? "true" : "false"; + break; + case BLANK: + case ERROR: + default: + value = ""; + break; + } + return value; + } catch (Exception e) { + return ""; + } + } + /** + * + * @param cell A cell in the Workbook + * @return A string representation of the cell. Replaces cell coordinates based on sheet offsets. + */ + private String getCellTextWithOffset(org.apache.poi.ss.usermodel.Cell cell) { + try { + String value; + org.apache.poi.ss.usermodel.CellType cellType = cell.getCellType(); + switch (cellType) { + case NUMERIC: + value = getNumericCellValue(cell); + break; + case STRING: + value = cell.getStringCellValue(); + break; + case FORMULA: + value = "=" + getFormulaWithOffset(cell); + break; + case BOOLEAN: + value = cell.getBooleanCellValue() ? "true" : "false"; + break; + case BLANK: + case ERROR: + default: + value = ""; + break; + } + return value; + } catch (Exception e) { + return e.getMessage(); + } + } + /** + * + * @param cell A cell from the Workbook + * @return A String representation of the numeric cell + */ + static private String getNumericCellValue(org.apache.poi.ss.usermodel.Cell cell) { + double numericValue = cell.getNumericCellValue(); + DataFormatter dataFormatter = new DataFormatter(); + try { + CellStyle cellStyle = cell.getCellStyle(); + return dataFormatter.formatRawCellContents(numericValue + , cellStyle.getDataFormat() + , cellStyle.getDataFormatString() + ); + } catch (Exception e) { + return Double.toString(numericValue); + } + } + + public static void main(String[] args) throws IOException { + //File xls = new File("F:\\Program Files\\ApacheTomcat\\apache-tomcat-9.0.41\\temp\\test.xls,test.xls.temp"); + File xls = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\test_xssf.xlsx"); + //File xls = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\test_hssf.xls"); + //File xls = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\albert_meyers_000_1_1.pst.0.xls"); + //File xls = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\cooper_richey_000_1_1.pst.256.xls"); + //File xls = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\craig_dean_000_1_1.pst.3.xls"); + //File xls = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\andrea_ring_000_1_1.pst.1.xls"); + + File csv = new File("F:\\Code\\dataspread\\xmlParser\\dataset-formula-comp-master\\SinglethreadedParser\\testDS\\result.csv"); + + //basicFullConvert(xls, csv); + + XLStoCSV tester = new XLStoCSV(xls); + tester.fullConvert(csv); + + } +} diff --git a/webapp/src/api/controller/ImportController.java b/webapp/src/api/controller/ImportController.java index b06fb50ee..c94802628 100644 --- a/webapp/src/api/controller/ImportController.java +++ b/webapp/src/api/controller/ImportController.java @@ -1,6 +1,7 @@ package api.controller; import api.JsonWrapper; +import api.XLStoCSV; import api.utils.ResumableInfo; import api.utils.ResumableInfoStorage; import org.model.AutoRollbackConnection; @@ -64,6 +65,21 @@ public HashMap uploadFile(@RequestParam("file") MultipartFile fi info.uploadedChunks.add(new ResumableInfo.ResumableChunkNumber(resumableChunkNumber)); if (info.checkIfUploadFinished()) { //Check if all chunks uploaded, and change filename ResumableInfoStorage.getInstance().remove(info); + String FilePath = new File(UPLOAD_DIR, resumableFilename).getAbsolutePath(); + int i = FilePath.lastIndexOf('.'); + String ext = FilePath.substring(i + 1); + if (ext.equals("xls") || ext.equals("xlsx")) { + File csv = new File(new File(UPLOAD_DIR, + resumableFilename).getAbsolutePath().substring(0, i) + ".csv"); + try { + XLStoCSV tester = new XLStoCSV(new File(FilePath)); + tester.fullConvert(csv); + System.out.println("File converted to CSV"); + } catch (Exception e) { + e.printStackTrace(); + } + resumableFilename = csv.getName(); + } System.out.println("File uploaded to " + UPLOAD_DIR); InputStream inputStream = new FileInputStream(new File(UPLOAD_DIR, resumableFilename)); HashMap msg = importBook(inputStream); From 94c1b0304d0af3ae680e8c7c724652184caa5d9f Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 9 Nov 2021 04:13:10 -0800 Subject: [PATCH 2/3] Changed TYPEFIVE to include gaps 1-5 --- .../src/CompressionSizeTest/SizeTestMain.java | 11 +- .../model/impl/sys/DependencyTableComp.java | 125 +++++++++--------- .../zss/model/impl/sys/utils/EdgeMeta.java | 19 ++- .../zss/model/impl/sys/utils/PatternType.java | 3 +- 4 files changed, 88 insertions(+), 70 deletions(-) diff --git a/testcode/src/CompressionSizeTest/SizeTestMain.java b/testcode/src/CompressionSizeTest/SizeTestMain.java index 0336c9ccd..289d4bfa6 100644 --- a/testcode/src/CompressionSizeTest/SizeTestMain.java +++ b/testcode/src/CompressionSizeTest/SizeTestMain.java @@ -21,10 +21,10 @@ public class SizeTestMain { /** * DB Connection Configuration * */ - public static String url = "jdbc:postgresql://127.0.0.1:5432/dbtest"; + public static String url = "jdbc:postgresql://127.0.0.1:5432/SizeTest"; public static String dbDriver = "org.postgresql.Driver"; - public static String userName = "totemtang"; - public static String password = "1234"; + public static String userName = "dataspreaduser"; + public static String password = "password"; public static void main (String[] args) { @@ -95,8 +95,9 @@ public static void main (String[] args) { int patternIdx = precMeta.getEdgeMeta().patternType.ordinal(); compEdges[patternIdx] = compEdges[patternIdx] + 1; - int compressCount = - precMeta.getPatternType() == PatternType.TYPEFIVE ? (dep.getCellCount() + 1)/2 : dep.getCellCount(); + int compressCount = (dep.getCellCount() + precMeta.getEdgeMeta().gapLength) / + (precMeta.getEdgeMeta().gapLength + 1); + uncompEdges[patternIdx] = uncompEdges[patternIdx] + compressCount; }); }); diff --git a/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java b/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java index a53bc358d..ad6f55b69 100644 --- a/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java +++ b/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java @@ -47,6 +47,8 @@ public class DependencyTableComp extends DependencyTableAdv { private CompressInfoComparator compressInfoComparator = new CompressInfoComparator(); + private final int MAX_GAP = 4; + public DependencyTableComp() {} public Map> getAllEdges() { @@ -162,7 +164,8 @@ private void flushUpdateCache(LinkedList updateCache, try { deleteDBEntry(dbContext, edgeUpdate.oldPrec, edgeUpdate.oldDep, edgeUpdate.oldEdgeMeta); insertDBEntry(dbContext, edgeUpdate.newPrec, edgeUpdate.newDep, edgeUpdate.newEdgeMeta.patternType, - edgeUpdate.newEdgeMeta.startOffset, edgeUpdate.newEdgeMeta.endOffset); + edgeUpdate.newEdgeMeta.startOffset, edgeUpdate.newEdgeMeta.endOffset, + edgeUpdate.newEdgeMeta.gapLength); } catch (SQLException e) { e.printStackTrace(); } @@ -185,7 +188,7 @@ private void fastModify(Ref prec, Ref dep, boolean isInsert, Ref newDep = selectedInfo.dep.getBoundingBox(selectedInfo.candDep); Pair offsetPair = computeOffset(newPrec, newDep, selectedInfo.compType); updateMatch.updateEdge(newPrec, newDep, - new EdgeMeta(selectedInfo.compType, offsetPair.getX(), offsetPair.getY())); + new EdgeMeta(selectedInfo.compType, updatePair.getX().edgeMeta.gapLength, offsetPair.getX(), offsetPair.getY())); updateCache.addFirst(updateMatch); } } else { @@ -199,11 +202,12 @@ private void fastModify(Ref prec, Ref dep, boolean isInsert, Pair offsetPair = computeOffset(newPrec, newDep, selectedInfo.compType); EdgeUpdate evicted = addToUpdateCache(updateCache, newPrec, newDep, - new EdgeMeta(selectedInfo.compType, offsetPair.getX(), offsetPair.getY())); + new EdgeMeta(selectedInfo.compType, selectedInfo.edgeMeta.gapLength, offsetPair.getX(), offsetPair.getY())); if (evicted != null) { deleteDBEntry(dbContext, evicted.oldPrec, evicted.oldDep, evicted.oldEdgeMeta); insertDBEntry(dbContext, evicted.newPrec, evicted.newDep, evicted.newEdgeMeta.patternType, - evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset); + evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset, + evicted.newEdgeMeta.gapLength); } } else { EdgeMeta noTypeEdgeMeta = new EdgeMeta(PatternType.NOTYPE, @@ -212,7 +216,8 @@ private void fastModify(Ref prec, Ref dep, boolean isInsert, if (evicted != null) { deleteDBEntry(dbContext, evicted.oldPrec, evicted.oldDep, evicted.oldEdgeMeta); insertDBEntry(dbContext, evicted.newPrec, evicted.newDep, evicted.newEdgeMeta.patternType, - evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset); + evicted.newEdgeMeta.startOffset, evicted.newEdgeMeta.endOffset, + evicted.newEdgeMeta.gapLength); } } } catch (SQLException e) { @@ -269,7 +274,7 @@ public void refreshCache(String bookName, String sheetName) { private void loadEverything(String bookName, String sheetName) { _rectToRefCache = RTree.create(); String selectQuery = - " SELECT range::box, dep_range::box, pattern_type, offsetRange::box" + + " SELECT range::box, dep_range::box, pattern_type, offsetRange::box, gap_length" + " FROM " + dependencyTableName + " WHERE bookname = ?" + " AND sheetname = ?"; @@ -292,8 +297,9 @@ private void loadEverything(String bookName, String sheetName) { (int) offsetRange.point[0].y); Offset endOffset = new Offset((int) offsetRange.point[1].x, (int) offsetRange.point[1].y); + int gapLength = rs.getInt(5); EdgeMeta edgeMeta = - new EdgeMeta(patternType, startOffset, endOffset); + new EdgeMeta(patternType, gapLength, startOffset, endOffset); insertMemEntry(prec, dep, edgeMeta); } } catch (SQLException e) { @@ -415,7 +421,7 @@ private void performOneInsert(Ref prec, Ref dep) { DBContext dbContext = new DBContext(connection); if (compressInfoList.isEmpty()) { insertDBEntry(dbContext, prec, dep, PatternType.NOTYPE, - Offset.noOffset, Offset.noOffset); + Offset.noOffset, Offset.noOffset, 0); } else { CompressInfo selectedInfo = Collections.min(compressInfoList, compressInfoComparator); @@ -436,7 +442,7 @@ private void updateOneCompressEntry(DBContext dbContext, Ref newDep = selectedInfo.dep.getBoundingBox(selectedInfo.candDep); Pair offsetPair = computeOffset(newPrec, newDep, selectedInfo.compType); insertDBEntry(dbContext, newPrec, newDep, selectedInfo.compType, - offsetPair.x, offsetPair.y); + offsetPair.x, offsetPair.y, 0); } private Pair findCompressInfoInUpdateCache(Ref prec, Ref dep, @@ -455,8 +461,8 @@ private Pair findCompressInfoInUpdateCache(Ref prec, R if (updatePair == null) { for (EdgeUpdate oneUpdate: updateCache) { - CompressInfo compRes = findCompressionPatternGapOne(prec, dep, - oneUpdate.newPrec, oneUpdate.newDep, oneUpdate.newEdgeMeta); + CompressInfo compRes = findCompressionPatternGapOneToN(prec, dep, + oneUpdate.newPrec, oneUpdate.newDep, oneUpdate.newEdgeMeta, MAX_GAP); PatternType compType = compRes.compType; if (compType != PatternType.NOTYPE) { updatePair = new Pair<>(compRes, oneUpdate); @@ -532,13 +538,15 @@ private void deleteDBEntry(DBContext dbContext, " AND sheetname = ?" + " AND range ~= ?" + " AND dep_range ~= ?" + - " AND offsetRange ~= ?"; + " AND offsetRange ~= ?" + + " AND gap_length = ?"; PreparedStatement retStmt = dbContext.getConnection().prepareStatement(query); retStmt.setString(1, prec.getBookName()); retStmt.setString(2, prec.getSheetName()); retStmt.setObject(3, RefUtils.refToPGBox(prec), Types.OTHER); retStmt.setObject(4, RefUtils.refToPGBox(dep), Types.OTHER); retStmt.setObject(5, RefUtils.offsetToPGBox(edgeMeta.startOffset, edgeMeta.endOffset), Types.OTHER); + retStmt.setInt(6, edgeMeta.gapLength); retStmt.execute(); } @@ -551,12 +559,13 @@ private void insertDBEntry(DBContext dbContext, Ref newDep, PatternType patternType, Offset startOffset, - Offset endOffset) throws SQLException { - insertMemEntry(newPrec, newDep, new EdgeMeta(patternType, startOffset, endOffset)); + Offset endOffset, + int gapLength) throws SQLException { + insertMemEntry(newPrec, newDep, new EdgeMeta(patternType, gapLength, startOffset, endOffset)); if (!refreshCacheMode) { String query = "INSERT INTO " + dependencyTableName + - " VALUES (?,?,?,?,?,?,?,?,?)"; + " VALUES (?,?,?,?,?,?,?,?,?,?)"; PreparedStatement retStmt = dbContext.getConnection().prepareStatement(query); retStmt.setString(1, newPrec.getBookName()); retStmt.setString(2, newPrec.getSheetName()); @@ -567,6 +576,7 @@ private void insertDBEntry(DBContext dbContext, retStmt.setBoolean(7, true); retStmt.setInt(8, patternType.ordinal()); retStmt.setObject(9, RefUtils.offsetToPGBox(startOffset, endOffset)); + retStmt.setInt(10, gapLength); retStmt.execute(); } @@ -653,7 +663,7 @@ private void performOneDelete(Ref delDep) { } else { try { insertDBEntry(dbContext, newPrec, newDep, newEdgeMeta.patternType, - newEdgeMeta.startOffset, newEdgeMeta.endOffset); + newEdgeMeta.startOffset, newEdgeMeta.endOffset, newEdgeMeta.gapLength); } catch (SQLException e) { e.printStackTrace(); } @@ -772,59 +782,49 @@ private CompressInfo findCompressionPattern(Ref prec, Ref dep, prec, dep, candPrec, candDep, metaData); } - private CompressInfo findCompressionPatternGapOne(Ref prec, Ref dep, - Ref candPrec, Ref candDep, EdgeMeta metaData) { - if (dep.getColumn() == candDep.getColumn() && candDep.getLastRow() - dep.getRow() == -2) { - if (metaData.patternType == PatternType.NOTYPE) { - Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); - Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true); - - Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); - Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false); - - if (offsetStartA.equals(offsetStartB) && - offsetEndA.equals(offsetEndB)) { - return new CompressInfo(false, Direction.TODOWN, PatternType.TYPEFIVE, - prec, dep, candPrec, candDep, metaData); - } - } else if (metaData.patternType == PatternType.TYPEFIVE) { - Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); - Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); - - if (offsetStartA.equals(metaData.startOffset) && - offsetEndA.equals(metaData.endOffset)) { - return new CompressInfo(false, Direction.TODOWN, PatternType.TYPEFIVE, - prec, dep, candPrec, candDep, metaData); - } + private CompressInfo findCompressionPatternGapOneToN(Ref prec, Ref dep, + Ref candPrec, Ref candDep, EdgeMeta metaData, int N) { + Direction patternDirection = Direction.NODIRECTION; + for (int gapLength = 1; gapLength <= N; gapLength++) { + if (dep.getColumn() == candDep.getColumn() && dep.getRow() - candDep.getLastRow() == gapLength + 1) { + patternDirection = Direction.TODOWN; + } else if (dep.getRow() == candDep.getRow() && dep.getColumn() - candDep.getLastColumn() == gapLength + 1) { + patternDirection = Direction.TORIGHT; } - } else if (dep.getRow() == candDep.getRow() && candDep.getLastColumn() - dep.getColumn() == -2) { - if (metaData.patternType == PatternType.NOTYPE) { - Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); - Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true); - - Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); - Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false); - - if (offsetStartA.equals(offsetStartB) && - offsetEndA.equals(offsetEndB)) { - return new CompressInfo(false, Direction.TORIGHT, PatternType.TYPEFIVE, - prec, dep, candPrec, candDep, metaData); - } - } else if (metaData.patternType == PatternType.TYPEFIVE) { - Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); - Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); - - if (offsetStartA.equals(metaData.startOffset) && - offsetEndA.equals(metaData.endOffset)) { - return new CompressInfo(false, Direction.TORIGHT, PatternType.TYPEFIVE, - prec, dep, candPrec, candDep, metaData); + if (patternDirection != Direction.NODIRECTION) { + if (metaData.patternType == PatternType.NOTYPE) { + Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); + Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true); + + Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); + Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false); + + if (offsetStartA.equals(offsetStartB) && + offsetEndA.equals(offsetEndB)) { + metaData.setGapLength(gapLength); + return new CompressInfo(false, patternDirection, PatternType.TYPEFIVE, + prec, dep, candPrec, candDep, metaData); + } + } else if (metaData.patternType == PatternType.TYPEFIVE) { + Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); + Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); + + if (offsetStartA.equals(metaData.startOffset) && + offsetEndA.equals(metaData.endOffset)) { + metaData.setGapLength(gapLength); + return new CompressInfo(false, patternDirection, PatternType.TYPEFIVE, + prec, dep, candPrec, candDep, metaData); + } } } + } + return new CompressInfo(false, Direction.NODIRECTION, PatternType.NOTYPE, prec, dep, candPrec, candDep, metaData); } + private PatternType findCompPatternHelper(Direction direction, Ref prec, Ref dep, Ref candPrec, Ref candDep, @@ -1099,7 +1099,8 @@ private class CompressInfo { Direction direction, PatternType compType, Ref prec, Ref dep, - Ref candPrec, Ref candDep, EdgeMeta edgeMeta) { + Ref candPrec, Ref candDep, + EdgeMeta edgeMeta) { this.isDuplicate = isDuplicate; this.direction = direction; this.compType = compType; diff --git a/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/EdgeMeta.java b/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/EdgeMeta.java index 91d2b6e1a..5c46a4cf2 100644 --- a/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/EdgeMeta.java +++ b/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/EdgeMeta.java @@ -7,10 +7,20 @@ public class EdgeMeta { public final PatternType patternType; public final Offset startOffset; public final Offset endOffset; + public int gapLength; public EdgeMeta(PatternType patternType, Offset startOffset, Offset endOffset) { this.patternType = patternType; + this.gapLength = 0; + this.startOffset = startOffset; + this.endOffset = endOffset; + } + + public EdgeMeta(PatternType patternType, int gapLength, + Offset startOffset, Offset endOffset) { + this.patternType = patternType; + this.gapLength = gapLength; this.startOffset = startOffset; this.endOffset = endOffset; } @@ -21,8 +31,13 @@ public boolean equals(Object o) { if (!(o instanceof EdgeMeta)) return false; EdgeMeta edgeMeta = (EdgeMeta) o; return patternType == edgeMeta.patternType && - Objects.equals(startOffset, edgeMeta.startOffset) && - Objects.equals(endOffset, edgeMeta.endOffset); + Objects.equals(this.startOffset, edgeMeta.startOffset) && + Objects.equals(this.endOffset, edgeMeta.endOffset) && + Objects.equals(this.gapLength, edgeMeta.gapLength); + } + + public void setGapLength(int gapLength) { + this.gapLength = gapLength; } @Override diff --git a/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternType.java b/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternType.java index d6f22c5b9..864708bef 100644 --- a/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternType.java +++ b/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternType.java @@ -6,6 +6,7 @@ public enum PatternType { TYPETWO, // Relative start, Absolute end TYPETHREE, // Absolute start, Relative end TYPEFOUR, // Absolute start, Absolute end - TYPEFIVE, // Relative + Relative with gap 1 + TYPEFIVE, // Relative + Relative with gap 1-5 + TYPESIX, // Absolute + Absolute with gap 1-5 NOTYPE } From c2cfe4367865d5717f327bf45c95b1dd033c0bd2 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Thu, 11 Nov 2021 22:21:22 -0800 Subject: [PATCH 3/3] Added TYPESIX (FF GapN); still a little buggy --- .../model/impl/sys/DependencyTableComp.java | 56 +++++++--------- .../model/impl/sys/utils/PatternTools.java | 65 ++++++++++++++----- 2 files changed, 72 insertions(+), 49 deletions(-) diff --git a/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java b/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java index ad6f55b69..cd4e7d8f2 100644 --- a/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java +++ b/zssmodel/src/org/zkoss/zss/model/impl/sys/DependencyTableComp.java @@ -784,43 +784,33 @@ private CompressInfo findCompressionPattern(Ref prec, Ref dep, private CompressInfo findCompressionPatternGapOneToN(Ref prec, Ref dep, Ref candPrec, Ref candDep, EdgeMeta metaData, int N) { - Direction patternDirection = Direction.NODIRECTION; + Direction direction = Direction.NODIRECTION; + PatternType pattern = PatternType.NOTYPE; + // if (dep.getColumn() == candDep.getColumn() && dep.getRow() - candDep.getLastRow() == gapLength + 1) { + if (dep.getColumn() == candDep.getColumn() && dep.getRow() != candDep.getRow()) { + direction = Direction.TODOWN; + //} else if (dep.getRow() == candDep.getRow() && dep.getColumn() - candDep.getLastColumn() == gapLength + 1) { + } else if (dep.getColumn() != candDep.getColumn() && dep.getRow() == candDep.getRow()) { + direction = Direction.TORIGHT; + } + if (direction == Direction.NODIRECTION) { + return new CompressInfo(false, Direction.NODIRECTION, PatternType.NOTYPE, + prec, dep, candPrec, candDep, metaData); + } for (int gapLength = 1; gapLength <= N; gapLength++) { - if (dep.getColumn() == candDep.getColumn() && dep.getRow() - candDep.getLastRow() == gapLength + 1) { - patternDirection = Direction.TODOWN; - } else if (dep.getRow() == candDep.getRow() && dep.getColumn() - candDep.getLastColumn() == gapLength + 1) { - patternDirection = Direction.TORIGHT; - } - if (patternDirection != Direction.NODIRECTION) { - if (metaData.patternType == PatternType.NOTYPE) { - Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); - Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true); - - Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); - Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false); - - if (offsetStartA.equals(offsetStartB) && - offsetEndA.equals(offsetEndB)) { - metaData.setGapLength(gapLength); - return new CompressInfo(false, patternDirection, PatternType.TYPEFIVE, - prec, dep, candPrec, candDep, metaData); - } - } else if (metaData.patternType == PatternType.TYPEFIVE) { - Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); - Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); - - if (offsetStartA.equals(metaData.startOffset) && - offsetEndA.equals(metaData.endOffset)) { - metaData.setGapLength(gapLength); - return new CompressInfo(false, patternDirection, PatternType.TYPEFIVE, - prec, dep, candPrec, candDep, metaData); - } - } + if (isCompressibleTypeFive(candPrec, candDep, prec, dep, direction, gapLength, metaData)) { + metaData.setGapLength(gapLength); + pattern = PatternType.TYPEFIVE; + break; + } else if (isCompressibleTypeSix(candPrec, candDep, prec, dep, direction, gapLength)) { + metaData.setGapLength(gapLength); + pattern = PatternType.TYPESIX; + break; } } - return new CompressInfo(false, Direction.NODIRECTION, PatternType.NOTYPE, + return new CompressInfo(false, direction, pattern, prec, dep, candPrec, candDep, metaData); } @@ -855,7 +845,7 @@ private Iterable findOverlapAndAdjacency(Ref ref) { findOverlappingRefs(ref).forEachRemaining(res::addLast); Arrays.stream(Direction.values()).filter(direction -> direction != Direction.NODIRECTION) .forEach(direction -> - findOverlappingRefs(shiftRef(ref, direction)) + findOverlappingRefs(shiftRef(ref, direction, 1)) .forEachRemaining(adjRef -> { if (isValidAdjacency(adjRef, ref)) res.addLast(adjRef); // valid adjacency }) diff --git a/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternTools.java b/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternTools.java index 032aed942..8eacfad38 100644 --- a/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternTools.java +++ b/zssmodel/src/org/zkoss/zss/model/impl/sys/utils/PatternTools.java @@ -2,6 +2,7 @@ import org.zkoss.util.Pair; import org.zkoss.zss.model.impl.RefImpl; +import org.zkoss.zss.model.impl.sys.DependencyTableComp; import org.zkoss.zss.model.sys.dependency.Ref; import java.util.Arrays; @@ -14,7 +15,7 @@ public class PatternTools { public static boolean isCompressibleTypeOne(Ref lastCandPrec, Ref prec, Direction direction) { - Ref shiftedRef = shiftRef(lastCandPrec, direction); + Ref shiftedRef = shiftRef(lastCandPrec, direction, 1); return shiftedRef != null && shiftedRef.equals(prec); } @@ -24,9 +25,9 @@ public static boolean isCompressibleTypeZero(Ref prec, Ref dep, boolean isTypeZero = false; for (Direction direction: Direction.values()) { if (direction != Direction.NODIRECTION && !isTypeZero) { - Ref shiftedRef = shiftRef(prec, direction); + Ref shiftedRef = shiftRef(prec, direction, 1); if (shiftedRef != null && shiftedRef.equals(dep)) { // check adjacency - Ref lastCandDep = shiftRef(lastCandPrec, direction); + Ref lastCandDep = shiftRef(lastCandPrec, direction, 1); isTypeZero = (lastCandDep != null && lastCandDep.equals(prec)) || lastCandPrec.equals(dep); } @@ -52,6 +53,42 @@ public static boolean isCompressibleTypeThree(Ref lastCandPrec, Ref prec, return isExtendedStart(lastCandPrec, prec, direction); } + public static boolean isCompressibleTypeFour(Ref lastCandPrec, Ref prec) { + return lastCandPrec.equals(prec); + } + + public static boolean isCompressibleTypeFive(Ref candPrec, Ref candDep, + Ref prec, Ref dep, + Direction direction, int gapLength, EdgeMeta metaData) { + if (dep.getColumn() == candDep.getColumn() && dep.getRow() - candDep.getLastRow() == gapLength + 1 || + dep.getRow() == candDep.getRow() && dep.getColumn() - candDep.getLastColumn() == gapLength + 1) { + if (metaData.patternType == PatternType.NOTYPE) { + Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); + Offset offsetStartB = RefUtils.refToOffset(candPrec, candDep, true); + + Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); + Offset offsetEndB = RefUtils.refToOffset(candPrec, candDep, false); + + return offsetStartA.equals(offsetStartB) && + offsetEndA.equals(offsetEndB); + } else if (metaData.patternType == PatternType.TYPEFIVE) { + Offset offsetStartA = RefUtils.refToOffset(prec, dep, true); + Offset offsetEndA = RefUtils.refToOffset(prec, dep, false); + + return offsetStartA.equals(metaData.startOffset) && + offsetEndA.equals(metaData.endOffset); + } + } + return false; + } + + public static boolean isCompressibleTypeSix(Ref candPrec, Ref candDep, + Ref prec, Ref dep, + Direction direction, int gapLength) { + Ref shiftedRef = shiftRef(candDep, direction, gapLength + 1); + return candPrec.equals(prec) && shiftedRef != null && shiftedRef.equals(dep); + } + public static boolean isExtendedEnd(Ref lastCandPrec, Ref prec, Direction direction) { if (direction == Direction.TODOWN) { @@ -112,40 +149,36 @@ public static boolean isExtendedStart(Ref lastCandPrec, Ref prec, } else return false; } - public static boolean isCompressibleTypeFour(Ref lastCandPrec, Ref prec) { - return lastCandPrec.equals(prec); - } - - public static Ref shiftRef(Ref ref, Direction direction) { + public static Ref shiftRef(Ref ref, Direction direction, int step) { Ref res = null; switch (direction) { case TOLEFT: if (ref.getColumn() != FIRST_COL) { res = new RefImpl(ref.getBookName(), ref.getSheetName(), - ref.getRow(), ref.getColumn() - SHIFT_STEP, - ref.getLastRow(), ref.getLastColumn() - SHIFT_STEP); + ref.getRow(), ref.getColumn() - step, + ref.getLastRow(), ref.getLastColumn() - step); } break; case TORIGHT: res = new RefImpl(ref.getBookName(), ref.getSheetName(), - ref.getRow(), ref.getColumn() + SHIFT_STEP, - ref.getLastRow(), ref.getLastColumn() + SHIFT_STEP); + ref.getRow(), ref.getColumn() + step, + ref.getLastRow(), ref.getLastColumn() + step); break; case TOUP: if (ref.getRow() != FIRST_ROW) { res = new RefImpl(ref.getBookName(), ref.getSheetName(), - ref.getRow() - SHIFT_STEP, ref.getColumn(), - ref.getLastRow() - SHIFT_STEP, ref.getLastColumn()); + ref.getRow() - step, ref.getColumn(), + ref.getLastRow() - step, ref.getLastColumn()); } break; default: // TODOWN res = new RefImpl(ref.getBookName(), ref.getSheetName(), - ref.getRow() + SHIFT_STEP, ref.getColumn(), - ref.getLastRow() + SHIFT_STEP, ref.getLastColumn()); + ref.getRow() + step, ref.getColumn(), + ref.getLastRow() + step, ref.getLastColumn()); } return res; }