organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-10 07:39:16 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 1

ICU-21569 Fix Java7 breakage in LSTM & add Java7 CI

Browse source 
See #1734
This commit is contained in:
Frank Yung-Fong Tang 2021-05-27 21:57:01 +00:00
parent 1d7b003a4e
commit a91cd6736e
3 changed files with 135 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
.github/workflows/icu_ci.yml vendored
View file

@ -30,6 +30,51 @@ jobs:
# Regex note: (?! ... ) is a negative lookahead. Succeed if the pattern is not present.
set +o pipefail && make doc 2>&1 | tee doxygen.log && ( ! grep -P 'warning:(?! .* file .?Doxyfile)' doxygen.log )
# Java7 ICU4J build and unit test
java7-icu4j-build-and-test:
runs-on: ubuntu-latest
steps:
- name: Checkout and setup
uses: actions/checkout@v2
with:
lfs: true
- name: Checkout lfs objects
run: git lfs pull
- uses: actions/setup-java@v1
with:
java-version: '8'
- name: Cache for Java 7 tarball
id: cache-java7
uses: actions/cache@v2
with:
path: java7-tarball
key: ${{ runner.os }}-java7-tarball
- name: Download Java 7
if: steps.cache-java7.outputs.cache-hit != 'true'
run: |
mkdir -p java7-tarball
download_url="https://download.java.net/openjdk/jdk7u75/ri/openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz"
wget -O java7-tarball/java_package.tar.gz $download_url
pushd java7-tarball
gunzip java_package.tar.gz
tar xvf java_package.tar
popd
- name: Configure Ant build to use Java 7 JRE
run: |
echo "java7.bootclasspath" > $RUNNER_TEMP/draft
ls java7-tarball/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft
paste -sd "=" < $RUNNER_TEMP/draft > icu4j/build-local.properties
- name: ICU4J
run: |
cd icu4j;
ant init;
ant check;
- name: List failures (if any)
run: |
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
if: ${{ failure() }}
# ICU4J build and unit test under Java11
java11-icu4j-build-and-test:
runs-on: ubuntu-latest

82
icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/LSTMBreakEngineTest.java
View file

@ -9,8 +9,6 @@ import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.text.CharacterIterator;
import java.text.StringCharacterIterator;
import java.util.Iterator;
import java.util.stream.Stream;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -66,52 +64,54 @@ public class LSTMBreakEngineTest extends TestFmwk {
errln("Could not open test data file " + filename);
return;
}
Stream<String> lines = (new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))).lines();
Iterator<String> iterator = lines.iterator();
BufferedReader br = (new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)));
int caseNum = 0;
String expected = "";
String actual = "";
LSTMBreakEngine engine = null;
while (iterator.hasNext()) {
String line = iterator.next();
String fields[] = line.split("\t");
if (fields[0].equals("Model:")) {
engine = createEngineFromTestData(fields[1], script);
} else if (fields[0].equals("Input:")) {
caseNum++;
int length = fields[1].length();
CharacterIterator input = new StringCharacterIterator(fields[1]);
DictionaryBreakEngine.DequeI foundBreaks = new DictionaryBreakEngine.DequeI();
int ret = engine.findBreaks(input, 0, length, foundBreaks);
StringBuilder sb = new StringBuilder();
sb.append('{');
for (int i = 0; i < foundBreaks.size(); i++) {
sb.append(foundBreaks.elementAt(i)).append(", ");
}
sb.append(length).append('}');
actual = sb.toString();
} else if (fields[0].equals("Output:")) {
StringBuilder sb = new StringBuilder();
int sep;
int start = 0;
int curr = 0;
sb.append('{');
while ((sep = fields[1].indexOf('|', start)) >= 0) {
int len = sep - start;
if (len > 0) {
if (curr > 0) {
sb.append(", ");
}
curr += len;
sb.append(curr);
String line;
try {
while ((line = br.readLine()) != null) {
String fields[] = line.split("\t");
if (fields[0].equals("Model:")) {
engine = createEngineFromTestData(fields[1], script);
} else if (fields[0].equals("Input:")) {
caseNum++;
int length = fields[1].length();
CharacterIterator input = new StringCharacterIterator(fields[1]);
DictionaryBreakEngine.DequeI foundBreaks = new DictionaryBreakEngine.DequeI();
int ret = engine.findBreaks(input, 0, length, foundBreaks);
StringBuilder sb = new StringBuilder();
sb.append('{');
for (int i = 0; i < foundBreaks.size(); i++) {
sb.append(foundBreaks.elementAt(i)).append(", ");
}
start = sep + 1;
sb.append(length).append('}');
actual = sb.toString();
} else if (fields[0].equals("Output:")) {
StringBuilder sb = new StringBuilder();
int sep;
int start = 0;
int curr = 0;
sb.append('{');
while ((sep = fields[1].indexOf('|', start)) >= 0) {
int len = sep - start;
if (len > 0) {
if (curr > 0) {
sb.append(", ");
}
curr += len;
sb.append(curr);
}
start = sep + 1;
}
sb.append('}');
expected = sb.toString();
assertEquals(line + " Test Case#" + caseNum , expected, actual);
}
sb.append('}');
expected = sb.toString();
assertEquals(line + " Test Case#" + caseNum , expected, actual);
}
} catch (IOException e) {
errln("Exception while reading lines of test data file " + filename + e.toString());
}
}
}

98
icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBILSTMTest.java
View file

@ -9,8 +9,6 @@ import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.text.CharacterIterator;
import java.text.StringCharacterIterator;
import java.util.Iterator;
import java.util.stream.Stream;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -56,63 +54,65 @@ public class RBBILSTMTest extends TestFmwk {
errln("Could not open test data file " + filename);
return;
}
Stream<String> lines = (new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))).lines();
Iterator<String> iterator = lines.iterator();
BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
int caseNum = 0;
String expected = "";
String actual = "";
LSTMBreakEngine engine = null;
while (iterator.hasNext()) {
String line = iterator.next();
String fields[] = line.split("\t");
if (fields[0].equals("Model:")) {
String actualModelName = LSTMBreakEngine.createData(script).fName;
if (!actualModelName.equals(fields[1])) {
errln("The name of the built in model " + actualModelName +
" does not match the model (" + fields[1] + ") expected for this test");
return;
}
} else if (fields[0].equals("Input:")) {
caseNum++;
int length = fields[1].length();
String input = "prefix " + fields[1] + " suffix";
bi.setText(input);
System.out.println("Input = " + input);
StringBuilder sb = new StringBuilder();
sb.append('{');
for (int bp = bi.first(); bp != BreakIterator.DONE; bp = bi.next()) {
sb.append(bp);
if (bp != input.length()) {
sb.append(", ");
String line;
try {
while ((line = br.readLine()) != null) {
String fields[] = line.split("\t");
if (fields[0].equals("Model:")) {
String actualModelName = LSTMBreakEngine.createData(script).fName;
if (!actualModelName.equals(fields[1])) {
errln("The name of the built in model " + actualModelName +
" does not match the model (" + fields[1] + ") expected for this test");
return;
}
}
sb.append('}');
actual = sb.toString();
} else if (fields[0].equals("Output:")) {
StringBuilder sb = new StringBuilder();
int sep;
int start = 0;
int curr = 0;
sb.append("{0, ");
String input = "prefix| |" + fields[1] + "| |suffix";
while ((sep = input.indexOf('|', start)) >= 0) {
int len = sep - start;
if (len > 0) {
if (curr > 0) {
} else if (fields[0].equals("Input:")) {
caseNum++;
int length = fields[1].length();
String input = "prefix " + fields[1] + " suffix";
bi.setText(input);
System.out.println("Input = " + input);
StringBuilder sb = new StringBuilder();
sb.append('{');
for (int bp = bi.first(); bp != BreakIterator.DONE; bp = bi.next()) {
sb.append(bp);
if (bp != input.length()) {
sb.append(", ");
}
curr += len;
sb.append(curr);
}
start = sep + 1;
sb.append('}');
actual = sb.toString();
} else if (fields[0].equals("Output:")) {
StringBuilder sb = new StringBuilder();
int sep;
int start = 0;
int curr = 0;
sb.append("{0, ");
String input = "prefix| |" + fields[1] + "| |suffix";
while ((sep = input.indexOf('|', start)) >= 0) {
int len = sep - start;
if (len > 0) {
if (curr > 0) {
sb.append(", ");
}
curr += len;
sb.append(curr);
}
start = sep + 1;
}
sb.append(", ").append(curr + input.length() - start);
sb.append('}');
expected = sb.toString();
assertEquals(input + " Test Case#" + caseNum , expected, actual);
actual = "";
}
sb.append(", ").append(curr + input.length() - start);
sb.append('}');
expected = sb.toString();
assertEquals(input + " Test Case#" + caseNum , expected, actual);
actual = "";
}
} catch (IOException e) {
errln("Exception while reading lines of test data file " + filename + e.toString());
}
}
}