organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-10 07:39:16 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 1

ICU-0 updates for script charts

Browse source 
X-SVN-Rev: 11928
This commit is contained in:
Mark Davis 2003-05-14 22:31:38 +00:00
parent 5f5a615f65
commit e7f2961b05
3 changed files with 545 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
tools/unicodetools/com/ibm/text/UCA/script_help.html Normal file
View file

@ -0,0 +1,31 @@
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="GENERATOR" content="Microsoft FrontPage 4.0">
<meta name="ProgId" content="FrontPage.Editor.Document">
<title>Chart Instructions</title>
<style>
<!--
th { background-color: #eeeeee }
-->
</style>
</head>
<body>
<h1>Instructions</h1>
<p>The Script charts provide an index to Unicode characters by script.</p>
<blockquote>
<p><i>To properly view these charts, your browser should be reasonably recent
so it handles Unicode and cascading style sheets, and you should install a
Unicode font and configure your browser to use it.</i></p>
</blockquote>
<p>Where the script = Common, the General Category is used in the index instead.</p>
</body>
</html>

35
tools/unicodetools/com/ibm/text/UCA/script_index_header.html Normal file
View file

@ -0,0 +1,35 @@
<!doctype HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"><html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta http-equiv="Content-Language" content="en-us">
<meta name="keywords" content="Basic">
<title>Script Chart</title>
<style><!--
p { font-size: 90% }
--></style>
<base target="main">
<link rel="stylesheet" type="text/css"
href="http://www.unicode.org/webscripts/standard_styles.css">
</head>
<body>
<table width="100%" cellpadding="0" cellspacing="0" border="0">
<tr>
<td colspan="2">
<table width="100%" border="0" cellpadding="0" cellspacing="0">
<tr>
<td class="icon"><a href="http://www.unicode.org/"><img border="0"
src="http://www.unicode.org/webscripts/logo60s2.gif" align="middle"
alt="[Unicode]" width="34" height="33"></a>&nbsp;&nbsp;<a class="bar"
href="http://www.unicode.org/unicode/faq/"><font size="3">Charts</font></a>
</tr>
</table>
</td>
</tr>
</table>
<div class="body">
<!-- BEGIN CONTENTS -->
<h2 align="center">Script Chart</h2>
<p align="center"><a href="help.html">Help</a>

479
tools/unicodetools/com/ibm/text/UCD/CompareProperties.java Normal file
View file

@ -0,0 +1,479 @@
/**
*******************************************************************************
* Copyright (C) 1996-2001, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*
* $Source: /xsrl/Nsvn/icu/unicodetools/com/ibm/text/UCD/CompareProperties.java,v $
* $Date: 2003/05/14 22:31:38 $
* $Revision: 1.1 $
*
*******************************************************************************
*/
package com.ibm.text.UCD;
import java.util.*;
import java.io.*;
import java.text.NumberFormat;
import com.ibm.text.utility.*;
import com.ibm.icu.text.UTF16;
import com.ibm.icu.text.UnicodeSet;
import com.ibm.icu.text.UnicodeSetIterator;
public class CompareProperties implements UCD_Types {
static final boolean DO_DISJOINT = false;
static CompareProperties me = null;
static void partition() throws IOException {
if (me == null) me = new CompareProperties();
me.printPartition();
}
static void statistics() throws IOException {
UnicodeSet a = new UnicodeSet("[abc]");
UnicodeSet empty = new UnicodeSet();
System.out.println(a.containsAll(empty));
System.out.println(empty.containsAll(a));
System.out.println(empty.containsAll(new UnicodeSet()));
if (me == null) me = new CompareProperties();
me.printStatistics();
}
public final class BitSetComparator implements Comparator {
public int compare(Object o1, Object o2) {
BitSet bs1 = (BitSet) o1;
BitSet bs2 = (BitSet) o2;
int count2 = bs1.size() > bs2.size() ? bs1.size() : bs2.size();
for (int i = 0; i < count2; ++i) {
if (bs1.get(i)) {
if (!bs2.get(i)) {
return 1;
}
} else if (bs2.get(i)) {
return -1;
}
}
return 0;
}
}
/*
*
* @author Davis
*
* Reverses the order of a comparison, for getting a list in reverse order
*/
public static class InverseComparator implements Comparator {
private Comparator other;
public InverseComparator(Comparator other) {
this.other = other;
}
public int compare(Object a, Object b) {
return other.compare(b, a);
}
}
/*
*
* @author Davis
*
* Reverses the order of a comparison, for getting a list in reverse order
*/
public static class MethodComparator implements Comparator {
public int compare(Object a, Object b) {
return ((Comparable)a).compareTo(b);
}
}
public final class UnicodeSetComparator implements Comparator {
/**
* Compares two UnicodeSets, producing a transitive ordering.
* @return -1 if first is smaller (in size) than second,
* 1 if first is greater (in size) than second,
* Otherwise (since they are equal in size)
* returns a comparison based on the first range that differs.
* If compareTo were added to UnicodeSet, this can be optimized to use list[i].
* @author Davis
*
*/
public int compare(Object o1, Object o2) {
UnicodeSet bs1 = (UnicodeSet) o1;
UnicodeSet bs2 = (UnicodeSet) o2;
if (bs1.size() < bs2.size()) return -1;
if (bs1.size() > bs2.size()) return 1;
UnicodeSetIterator it1 = new UnicodeSetIterator(bs1);
UnicodeSetIterator it2 = new UnicodeSetIterator(bs2);
// Note: because they are the same size, and we stop if any ranges
// are different, it is safe to test for both at the same time
while (it1.nextRange() && it2.nextRange()) {
if (it1.codepoint < it2.codepoint) return -1;
if (it1.codepoint > it2.codepoint) return 1;
if (it1.codepointEnd < it2.codepointEnd) return -1;
if (it1.codepointEnd > it2.codepointEnd) return 1;
}
return 0;
}
}
boolean isPartitioned = false;
UnicodeProperty[] props = new UnicodeProperty[500];
UnicodeSet[] sets = new UnicodeSet[500];
int count = 0;
BitSet[] disjoints = new BitSet[500];
BitSet[] contains = new BitSet[500];
BitSet[] isin = new BitSet[500];
BitSet[] equals = new BitSet[500];
Map map = new TreeMap(new BitSetComparator());
{
getProperties();
fillPropertyValues();
Utility.fixDot();
}
private void fillPropertyValues() {
BitSet probe = new BitSet();
int total = 0;
for (int cp = 0; cp <= 0x10FFFF; ++cp) {
Utility.dot(cp);
int cat = Default.ucd.getCategory(cp);
// if (cat == UNASSIGNED || cat == PRIVATE_USE || cat == SURROGATE) continue;
if (!Default.ucd.isAllocated(cp)) continue;
for (int i = 0; i < count; ++i) {
UnicodeProperty up = props[i];
boolean iProp = up.hasValue(cp);
if (iProp) {
probe.set(i);
sets[i].add(cp);
} else {
probe.clear(i);
}
}
++total;
UnicodeSet value = (UnicodeSet) map.get(probe);
if (value == null) {
value = new UnicodeSet();
map.put(probe.clone(), value);
// Utility.fixDot();
// System.out.println("Set Size: " + map.size() + ", total: " + total + ", " + Default.ucd.getCodeAndName(cp));
}
value.add(cp);
}
}
private void getProperties() {
Default.setUCD();
for (int i = 0; i < LIMIT_ENUM; ++i) { // || iType == SCRIPT
int iType = i & 0xFF00;
if (iType == AGE || iType == JOINING_GROUP || iType == COMBINING_CLASS) continue;
if (i == 0x0900) {
System.out.println("debug");
}
UnicodeProperty up = UnifiedBinaryProperty.make(i, Default.ucd);
if (up == null) continue;
if (up.getValueType() < BINARY) {
System.out.println("\tSkipping " + up.getName() + "; value varies");
continue;
}
if (!up.isStandard()) {
System.out.println("\tSkipping " + getPropName(up) + "; not standard");
continue;
}
if (up.getName(LONG).startsWith("Other_")) {
System.out.println("\tSkipping " + getPropName(up) + "; contributory");
continue;
}
if (up.isDefaultValue() || up.skipInDerivedListing()) {
System.out.println("\tSkipping " + getPropName(up) + "; default value");
continue;
}
// System.out.println(Utility.hex(i) + " " + up.getName(LONG) + "(" + up.getName(SHORT) + ")");
// System.out.println("\t" + up.getValue(LONG) + "(" + up.getValue(SHORT) + ")");
sets[count] = new UnicodeSet();
disjoints[count] = new BitSet();
equals[count] = new BitSet();
contains[count] = new BitSet();
isin[count] = new BitSet();
props[count++] = up;
System.out.println(Utility.hex(i) + " " + (count - 1) + " " + getPropName(count - 1));
}
System.out.println("props: " + count);
}
public void printPartition() throws IOException {
System.out.println("Set Size: " + map.size());
PrintWriter output = Utility.openPrintWriter("Partition"
+ GenerateData.getFileSuffix(true), Utility.LATIN1_WINDOWS);
Iterator it = map.keySet().iterator();
while (it.hasNext()) {
BitSet probe2 = (BitSet) it.next();
UnicodeSet value = (UnicodeSet) map.get(probe2);
output.println();
output.println(value);
output.println("Size: " + value.size());
for (int i = 0; i < count; ++i) {
if (!probe2.get(i)) continue;
output.print(" " + getPropName(i));
}
output.println();
}
output.println("Count: " + map.keySet().size());
output.close();
}
static final NumberFormat percent = NumberFormat.getPercentInstance(Locale.ENGLISH);
public void printStatistics() throws IOException {
System.out.println("Set Size: " + map.size());
PrintWriter output = Utility.openPrintWriter("Statistics"
+ GenerateData.getFileSuffix(true), Utility.LATIN1_WINDOWS);
System.out.println("Finding disjoints/contains");
for (int i = 0; i < count; ++i) {
System.out.println(getPropName(i));
for (int j = 0; j < count; ++j) {
if (j == i) continue;
if (i == 1 && j == 2) {
System.out.println("debug");
}
if (sets[i].containsNone(sets[j])) {
disjoints[i].set(j);
} else if (sets[i].equals(sets[j])) {
equals[i].set(j);
} else if (sets[i].containsAll(sets[j])) {
contains[i].set(j);
} else if (sets[j].containsAll(sets[i])) {
isin[i].set(j);
}
}
}
System.out.println("Removing non-maximal sets");
// a set is non-maximal if it is contained in one of the other sets
// so remove anything that is contained in one of the items
if (false) {
BitSet[] tempContains = new BitSet[count];
for (int i = 0; i < count; ++i) {
System.out.println(getPropName(i));
tempContains[i] = (BitSet) contains[i]; // worry about collisions
BitSet b = contains[i];
for (int j = 0; j < b.size(); ++j) {
if (b.get(j)) tempContains[i].andNot(contains[j]);
}
b = disjoints[i]; // don't worry
for (int j = 0; j < b.size(); ++j) {
if (b.get(j)) b.andNot(contains[j]);
}
}
for (int i = 0; i < count; ++i) {
contains[i] = tempContains[i];
}
}
System.out.println("Printing disjoints & contains");
// a set is non-maximal if it is contained in one of the other sets
// so remove anything that is contained in one of the items
List remainder = new ArrayList();
Map m = new TreeMap(); // new UnicodeSetComparator()
for (int i = 0; i < count; ++i) {
m.put(getPropName(i), new Integer(i)); // sets[i]
}
Iterator it = m.keySet().iterator();
while (it.hasNext()) {
Object key = it.next();
int index = ((Integer)m.get(key)).intValue();
boolean haveName = printBitSet(output, index, "EQUALS: ", equals[index], false);
haveName = printBitSet(output, index, "CONTAINS: ", contains[index], haveName);
haveName = printBitSet(output, index, "IS CONTAINED IN: ", isin[index], haveName);
if (DO_DISJOINT) {
printBitSet(output, index, "IS DISJOINT WITH: ", disjoints[index], haveName);
}
if (!haveName) remainder.add(getPropName(index));
}
it = remainder.iterator();
output.println();
output.print("NONE OF THE ABOVE: ");
boolean first = true;
while (it.hasNext()) {
Object key = it.next();
if (!first) output.print(", ");
first = false;
output.print(key);
}
output.println();
output.close();
}
private boolean printBitSet(PrintWriter output, int index, String title, BitSet b, boolean haveName) {
if (!b.isEmpty()) {
if (!haveName) {
output.println();
output.println(getPropName(index));
haveName = true;
}
output.print(title);
Set ss = new TreeSet();
for (int j = 0; j < b.size(); ++j) {
if (b.get(j)) {
ss.add(getPropName(j));
}
}
Iterator it = ss.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) output.print(", ");
first = false;
output.print(it.next());
}
output.println();
output.flush();
}
return haveName;
}
/*
UnicodeSet a_b = new UnicodeSet();
UnicodeSet ab = new UnicodeSet();
UnicodeSet _ab = new UnicodeSet();
*/
/*
a_b.set(sets[i]).removeAll(sets[j]);
ab.set(sets[i]).retainAll(sets[j]);
_ab.set(sets[j]).removeAll(sets[i]);
// we are interested in cases where a contains b or is contained by b
// contain = _ab = 0
// is contained == a_b = 0
// is disjoint == ab == 0
// is equal == contains & iscontained
double total = a_b.size() + ab.size() + _ab.size();
double limit = total*0.03;
boolean gotName = showDiff(output, "C", j, a_b, total, limit, false);
gotName = showDiff(output, "D", j, ab, total, limit, gotName);
gotName = showDiff(output, "S", j, _ab, total, limit, gotName);
if (gotName) output.println();
*/
private boolean showDiff(PrintWriter output, String title, int propIndex, UnicodeSet a_b,
double total, double limit, boolean gotName) {
if (0 < a_b.size() && a_b.size() < limit) {
if (!gotName) {
gotName = true;
output.print("\t" + getPropName(propIndex));
}
output.print("\t" + title + percent.format(a_b.size()/total));
}
return gotName;
}
private String getPropName(int propertyIndex) {
return getPropName(props[propertyIndex]);
}
private String getPropName(UnicodeProperty ubp) {
return Utility.getUnskeleton(ubp.getFullName(LONG), true);
}
public static void listDifferences() throws IOException {
Default.setUCD();
PrintWriter output = Utility.openPrintWriter("PropertyDifferences" + GenerateData.getFileSuffix(true), Utility.LATIN1_UNIX);
output.println("# Listing of relationships among properties, suitable for analysis by spreadsheet");
output.println("# Generated for " + Default.ucd.getVersion());
output.println(GenerateData.generateDateLine());
output.println("# P1 P2 R(P1,P2) C(P1&P2) C(P1-P2) C(P2-P1)");
for (int i = 1; i < UCD_Types.LIMIT_ENUM; ++i) {
int iType = i & 0xFF00;
if (iType == UCD_Types.JOINING_GROUP || iType == UCD_Types.AGE || iType == UCD_Types.COMBINING_CLASS || iType == UCD_Types.SCRIPT) continue;
UnicodeProperty upi = UnifiedBinaryProperty.make(i, Default.ucd);
if (upi == null) continue;
if (!upi.isStandard()) {
System.out.println("Skipping " + upi.getName() + "; not standard");
continue;
}
if (upi.getValueType() < UCD_Types.BINARY) {
System.out.println("Skipping " + upi.getName() + "; value varies");
continue;
}
String iNameShort = upi.getFullName(UCD_Types.SHORT);
String iNameLong = upi.getFullName(UCD_Types.LONG);
System.out.println();
System.out.println();
System.out.println(iNameLong);
output.println("#" + iNameLong);
int last = -1;
for (int j = i+1; j < UCD_Types.LIMIT_ENUM; ++j) {
int jType = j & 0xFF00;
if (jType == UCD_Types.JOINING_GROUP || jType == UCD_Types.AGE || jType == UCD_Types.COMBINING_CLASS || jType == UCD_Types.SCRIPT
|| (jType == iType && jType != UCD_Types.BINARY_PROPERTIES)) continue;
UnicodeProperty upj = UnifiedBinaryProperty.make(j, Default.ucd);
if (upj == null) continue;
if (!upj.isStandard()) continue;
if (upj.getValueType() < UCD_Types.BINARY) continue;
if ((j >> 8) != last) {
last = j >> 8;
System.out.println();
System.out.print("\t" + UCD_Names.SHORT_UNIFIED_PROPERTIES[last]);
output.flush();
output.println("#\t" + UCD_Names.SHORT_UNIFIED_PROPERTIES[last]);
} else {
System.out.print('.');
}
System.out.flush();
int bothCount = 0, i_jPropCount = 0, j_iPropCount = 0, iCount = 0, jCount = 0;
for (int cp = 0; cp <= 0x10FFFF; ++cp) {
int cat = Default.ucd.getCategory(cp);
if (cat == UCD_Types.UNASSIGNED || cat == UCD_Types.PRIVATE_USE || cat == UCD_Types.SURROGATE) continue;
if (!Default.ucd.isAllocated(cp)) continue;
boolean iProp = upi.hasValue(cp);
boolean jProp = upj.hasValue(cp);
if (jProp) ++jCount;
if (iProp) {
++iCount;
if (jProp) ++bothCount;
else ++i_jPropCount;
} else if (jProp) ++j_iPropCount;
}
if (iCount == 0 || jCount == 0) continue;
String jNameShort = upj.getFullName(UCD_Types.SHORT);
//String jNameLong = ubp.getFullID(j, LONG);
String rel = bothCount == 0 ? "DISJOINT"
: i_jPropCount == 0 && j_iPropCount == 0 ? "EQUALS"
: i_jPropCount == 0 ? "CONTAINS" // depends on reverse output
: j_iPropCount == 0 ? "CONTAINS"
: "OVERLAPS";
if (j_iPropCount > i_jPropCount) {
// reverse output
output.println(jNameShort + "\t" + iNameShort + "\t" + rel
+ "\t" + bothCount + "\t" + j_iPropCount + "\t" + i_jPropCount);
} else {
output.println(iNameShort + "\t" + jNameShort + "\t" + rel
+ "\t" + bothCount + "\t" + i_jPropCount + "\t" + j_iPropCount);
}
}
}
output.close();
}
}