Skip to content

Commit e0890a4

Browse files
wadimwwadimw
committed
Split integrity check step into document-level and TU-level steps
1 parent 7b84196 commit e0890a4

File tree

3 files changed

+74
-39
lines changed

3 files changed

+74
-39
lines changed

‎webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/DocumentIntegrityCheckStep.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package com.box.l10n.mojito.service.assetintegritychecker.integritychecker;
2+
3+
import com.google.common.io.CharStreams;
4+
import java.io.IOException;
5+
import java.io.Reader;
6+
import java.util.List;
7+
import net.sf.okapi.common.Event;
8+
import net.sf.okapi.common.pipeline.BasePipelineStep;
9+
import net.sf.okapi.common.resource.RawDocument;
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
12+
import org.springframework.beans.factory.annotation.Autowired;
13+
import org.springframework.beans.factory.annotation.Configurable;
14+
15+
/**
16+
* @author aloison
17+
*/
18+
@Configurable
19+
public class DocumentIntegrityCheckStep extends BasePipelineStep {
20+
21+
/** Logger */
22+
static Logger logger = LoggerFactory.getLogger(DocumentIntegrityCheckStep.class);
23+
24+
@Autowired IntegrityCheckerFactory integrityCheckerFactory;
25+
26+
@Override
27+
public String getName() {
28+
return "Document Integrity Check";
29+
}
30+
31+
@Override
32+
public String getDescription() {
33+
return "Runs document-level integrity checks to make sure a document is valid."
34+
+ " When an issue is detected, it will throw an IntegrityCheckException."
35+
+ " Expects: raw document. Sends back: raw document.";
36+
}
37+
38+
@Override
39+
protected Event handleRawDocument(Event event) {
40+
logger.debug("Check integrity of document");
41+
RawDocument document = event.getRawDocument();
42+
43+
String documentContent;
44+
try {
45+
Reader reader = document.getReader();
46+
documentContent = CharStreams.toString(reader);
47+
// CharStreams#toString does not close the readable implicitly
48+
reader.close();
49+
} catch (IOException e) {
50+
logger.error("Error reading document content", e);
51+
throw new RuntimeException("Error reading document content", e);
52+
}
53+
54+
// TODO(P1): do not hardcode the type here
55+
List<DocumentIntegrityChecker> documentIntegrityCheckers =
56+
integrityCheckerFactory.getDocumentCheckers("xliff");
57+
for (DocumentIntegrityChecker checker : documentIntegrityCheckers) {
58+
checker.check(documentContent);
59+
}
60+
61+
return super.handleRawDocument(event);
62+
}
63+
}

webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/IntegrityCheckStep.java renamed to webapp/src/main/java/com/box/l10n/mojito/service/assetintegritychecker/integritychecker/TextUnitIntegrityCheckStep.java

Lines changed: 4 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,7 @@
44
import com.box.l10n.mojito.entity.TMTextUnit;
55
import com.box.l10n.mojito.entity.TMTextUnitVariantComment;
66
import com.box.l10n.mojito.service.tm.TMTextUnitRepository;
7-
import com.google.common.io.CharStreams;
8-
import java.io.IOException;
97
import java.util.HashMap;
10-
import java.util.List;
118
import java.util.Map;
129
import java.util.Set;
1310
import net.sf.okapi.common.Event;
@@ -16,7 +13,6 @@
1613
import net.sf.okapi.common.pipeline.annotations.StepParameterMapping;
1714
import net.sf.okapi.common.pipeline.annotations.StepParameterType;
1815
import net.sf.okapi.common.resource.ITextUnit;
19-
import net.sf.okapi.common.resource.RawDocument;
2016
import net.sf.okapi.common.resource.TextContainer;
2117
import org.slf4j.Logger;
2218
import org.slf4j.LoggerFactory;
@@ -27,10 +23,10 @@
2723
* @author aloison
2824
*/
2925
@Configurable
30-
public class IntegrityCheckStep extends BasePipelineStep {
26+
public class TextUnitIntegrityCheckStep extends BasePipelineStep {
3127

3228
/** Logger */
33-
static Logger logger = LoggerFactory.getLogger(IntegrityCheckStep.class);
29+
static Logger logger = LoggerFactory.getLogger(TextUnitIntegrityCheckStep.class);
3430

3531
@Autowired TMTextUnitRepository tmTextUnitRepository;
3632

@@ -39,50 +35,22 @@ public class IntegrityCheckStep extends BasePipelineStep {
3935
Map<Long, Set<TextUnitIntegrityChecker>> textUnitIntegrityCheckerMap = new HashMap<>();
4036

4137
private LocaleId targetLocale;
42-
private RawDocument rawDocument;
4338

4439
@SuppressWarnings("deprecation")
4540
@StepParameterMapping(parameterType = StepParameterType.TARGET_LOCALE)
4641
public void setTargetLocale(LocaleId targetLocale) {
4742
this.targetLocale = targetLocale;
4843
}
4944

50-
@StepParameterMapping(parameterType = StepParameterType.INPUT_RAWDOC)
51-
public void setInputDocument(RawDocument rawDocument) {
52-
this.rawDocument = rawDocument;
53-
}
54-
5545
@Override
5646
public String getName() {
57-
return "Integrity Check";
47+
return "Text Unit Integrity Check";
5848
}
5949

6050
@Override
6151
public String getDescription() {
6252
return "Updates the TM with the extracted new/changed variants."
63-
+ " Expects: raw document. Sends back: original events.";
64-
}
65-
66-
@Override
67-
protected Event handleStartDocument(Event event) {
68-
logger.debug("Check integrity of document");
69-
70-
String documentContent = null;
71-
try {
72-
documentContent = CharStreams.toString(rawDocument.getReader());
73-
} catch (IOException e) {
74-
logger.error("Error reading document content", e);
75-
throw new RuntimeException("Error reading document content", e);
76-
}
77-
78-
// TODO(P1): do not hardcode the type here
79-
List<DocumentIntegrityChecker> documentIntegrityCheckers =
80-
integrityCheckerFactory.getDocumentCheckers("xliff");
81-
for (DocumentIntegrityChecker checker : documentIntegrityCheckers) {
82-
checker.check(documentContent);
83-
}
84-
85-
return super.handleStartDocument(event);
53+
+ " Expects: Text unit events. Sends back: original events.";
8654
}
8755

8856
@Override

‎webapp/src/main/java/com/box/l10n/mojito/service/tm/TMService.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@
4747
import com.box.l10n.mojito.security.AuditorAwareImpl;
4848
import com.box.l10n.mojito.service.WordCountService;
4949
import com.box.l10n.mojito.service.asset.AssetRepository;
50-
import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.IntegrityCheckStep;
50+
import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.DocumentIntegrityCheckStep;
51+
import com.box.l10n.mojito.service.assetintegritychecker.integritychecker.TextUnitIntegrityCheckStep;
5152
import com.box.l10n.mojito.service.locale.LocaleService;
5253
import com.box.l10n.mojito.service.pollableTask.InjectCurrentTask;
5354
import com.box.l10n.mojito.service.pollableTask.Pollable;
@@ -887,11 +888,14 @@ private UpdateTMWithXLIFFResult updateTMWithXliff(
887888
logger.debug("Configuring pipeline for localized XLIFF processing");
888889

889890
IPipelineDriver driver = new PipelineDriver();
891+
892+
driver.addStep(new DocumentIntegrityCheckStep());
893+
890894
driver.addStep(new RawDocumentToFilterEventsStep(new XLIFFFilter()));
891895

896+
driver.addStep(new TextUnitIntegrityCheckStep());
897+
892898
driver.addStep(getConfiguredQualityStep());
893-
IntegrityCheckStep integrityCheckStep = new IntegrityCheckStep();
894-
driver.addStep(integrityCheckStep);
895899

896900
abstractImportTranslationsStep.setImportWithStatus(importStatus);
897901
driver.addStep(abstractImportTranslationsStep);

0 commit comments

Comments
 (0)