kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

Review fixes.

Browse source
This commit is contained in:
Maksim Andrianov 2020-06-01 17:25:18 +03:00 • committed by mpimenov
parent 9800a56416
commit dbb19b2fdb
12 changed files with 162 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
tools/python/maps_generator/checks/check.py
View file

@ -48,8 +48,55 @@ def get_rel(r: ResLine) -> bool:
class Check(ABC):
"""
Base class for any checks.
Usual flow:
# Create check object.
check = AnyCheck("ExampleCheck")
# Do work.
check.check()
# Get results and process them
raw_result = check.get_result()
process_result(raw_result)
# or print result
check.print()
"""
def __init__(self, name: str):
self.name = name
def print(self, silent_if_no_results=False, filt=None, file=sys.stdout):
s = self.formatted_string(silent_if_no_results, filt)
if s:
print(s, file=file)
@abstractmethod
def check(self):
"""
Performs a logic of the check.
"""
pass
@abstractmethod
def get_result(self) -> Any:
"""
Returns a raw result of the check.
"""
pass
@abstractmethod
def formatted_string(self, silent_if_no_results=False, *args, **kwargs) -> str:
"""
Returns a formatted string of a raw result of the check.
"""
pass
class CompareCheckBase(Check, ABC):
def __init__(self, name: str):
super().__init__(name)
self.op: Callable[
[Any, Any], Any
] = lambda previous, current: current - previous
@ -77,25 +124,8 @@ class Check(ABC):
def set_filt(self, filt: Callable[[Any], bool]):
self.filt = filt
def print(self, silent_if_no_results=False, filt=None, file=sys.stdout):
s = self.formatted_string(silent_if_no_results, filt)
if s:
print(s, file=file)
@abstractmethod
def check(self):
pass
@abstractmethod
def get_result(self) -> Any:
pass
@abstractmethod
def formatted_string(self, silent_if_no_results=False, *args, **kwargs) -> str:
pass
class CompareCheck(Check):
class CompareCheck(CompareCheckBase):
def __init__(
self, name: str, old: Any, new: Any,
):
@ -156,7 +186,7 @@ class CompareCheck(Check):
)
class CompareCheckSet(Check):
class CompareCheckSet(CompareCheckBase):
def __init__(self, name: str):
super().__init__(name)
@ -215,15 +245,15 @@ class CompareCheckSet(Check):
lines = []
if no_results:
lines.append(f"{' ' * _offset}No results.")
lines.append(f"{' ' * (_offset + 2)}No results.")
for c in checks:
s = c.formatted_string(silent_if_no_results, filt, _offset + 1)
s = c.formatted_string(silent_if_no_results, filt, _offset + 2)
if s:
lines.append(f"{' ' * _offset + ' '}{s}")
lines.append(f"{' ' * (_offset + 2)}{s}")
for s in sets:
s = s.formatted_string(silent_if_no_results, filt, _offset + 1)
s = s.formatted_string(silent_if_no_results, filt, _offset + 2)
if s:
lines.append(s)

6
tools/python/maps_generator/checks/check_addresses.py
View file

@ -13,9 +13,13 @@ ADDR_PATTERN = re.compile(
def get_addresses_check_set(old_path: str, new_path: str) -> check.CompareCheckSet:
"""
Returns an addresses check set, that checks a difference in 'matched_percent'
addresses of BuildAddressTable between old logs and new logs.
"""
def do(path: str):
log = logs_reader.Log(path)
if not log.is_country_log:
if not log.is_mwm_log:
return None
found = logs_reader.find_and_parse(log.lines, ADDR_PATTERN)

6
tools/python/maps_generator/checks/check_categories.py
View file

@ -32,7 +32,11 @@ def parse_groups(path):
def get_categories_check_set(
old_path: str, new_path: str, categories_path: str
) -> check.CompareCheckSet:
cs = check.CompareCheckSet("Sections categories check")
"""
Returns a categories check set, that checks a difference in a number of
objects of categories(from categories.txt) between old mwms and new mwms.
"""
cs = check.CompareCheckSet("Categories check")
def make_do(indexes):
def do(path):

5
tools/python/maps_generator/checks/check_log_levels.py
View file

@ -13,6 +13,11 @@ def _get_log_stages(path):
def get_log_levels_check_set(old_path: str, new_path: str) -> check.CompareCheckSet:
"""
Returns a log levels check set, that checks a difference in a number of
message levels from warning and higher for each stage between old mwms
and new mwms.
"""
cs = check.CompareCheckSet("Log levels check")
def make_do(level, stage_name, cache={}):

24
tools/python/maps_generator/checks/check_mwm_types.py
View file

@ -19,23 +19,31 @@ def count_all_types(path: str):
def get_mwm_type_check_set(
old_path: str, new_path: str, _type: Union[str, int]
old_path: str, new_path: str, type_: Union[str, int]
) -> check.CompareCheckSet:
if isinstance(_type, str):
_type = type_index(_type)
assert _type >= 0, _type
"""
Returns a mwm type check set, that checks a difference in a number of
type [type_] between old mwms and new mwms.
"""
if isinstance(type_, str):
type_ = type_index(type_)
assert type_ >= 0, type_
return check.build_check_set_for_files(
f"Types check [{readable_type(_type)}]",
f"Types check [{readable_type(type_)}]",
old_path,
new_path,
ext=".mwm",
do=lambda path: count_all_types(path)[_type],
do=lambda path: count_all_types(path)[type_],
)
def get_mwm_all_types_check_set(old_path: str, new_path: str) -> check.CompareCheckSet:
cs = check.CompareCheckSet("Sections categories check")
def get_all_mwm_types_check_set(old_path: str, new_path: str) -> check.CompareCheckSet:
"""
Returns a all mwm types check set, that checks a difference in a number of
each type between old mwms and new mwms.
"""
cs = check.CompareCheckSet("Mwm types check")
def make_do(index):
return lambda path: count_all_types(path)[index]

20
tools/python/maps_generator/checks/check_sections.py
View file

@ -10,7 +10,9 @@ class SectionNames:
self.sections = sections
def __sub__(self, other):
return SectionNames(self.sections - other.sections)
return SectionNames(
{k: self.sections[k] for k in set(self.sections) - set(other.sections)}
)
def __lt__(self, other):
if isinstance(other, int):
@ -46,8 +48,8 @@ def get_appeared_sections_check_set(
old_path,
new_path,
ext=".mwm",
do=lambda path: SectionNames(read_sections(path).keys()),
diff_format=lambda s: ",".join(s.sections),
do=lambda path: SectionNames(read_sections(path)),
diff_format=lambda s: ", ".join(f"{k}:{v.size}" for k, v in s.sections.items()),
format=lambda s: f"number of sections: {len(s.sections)}",
)
@ -60,9 +62,9 @@ def get_disappeared_sections_check_set(
old_path,
new_path,
ext=".mwm",
do=lambda path: SectionNames(read_sections(path).keys()),
do=lambda path: SectionNames(read_sections(path)),
op=lambda previous, current: previous - current,
diff_format=lambda s: ",".join(s.sections),
diff_format=lambda s: ", ".join(f"{k}:{v.size}" for k, v in s.sections.items()),
format=lambda s: f"number of sections: {len(s.sections)}",
)
@ -70,6 +72,10 @@ def get_disappeared_sections_check_set(
def get_sections_existence_check_set(
old_path: str, new_path: str
) -> check.CompareCheckSet:
"""
Returns a sections existence check set, that checks appeared and
disappeared sections between old mwms and new mwms.
"""
cs = check.CompareCheckSet("Sections existence check")
cs.add_check(get_appeared_sections_check_set(old_path, new_path))
cs.add_check(get_disappeared_sections_check_set(old_path, new_path))
@ -86,6 +92,10 @@ def _get_sections_set(path):
def get_sections_size_check_set(old_path: str, new_path: str) -> check.CompareCheckSet:
"""
Returns a sections size check set, that checks a difference in a size
of each sections of mwm between old mwms and new mwms.
"""
sections_set = _get_sections_set(old_path)
sections_set.update(_get_sections_set(new_path))

4
tools/python/maps_generator/checks/check_size.py
View file

@ -4,6 +4,10 @@ from maps_generator.checks import check
def get_size_check_set(old_path: str, new_path: str) -> check.CompareCheckSet:
"""
Returns a size check set, that checks a difference in a size of mwm between
old mwms and new mwms.
"""
return check.build_check_set_for_files(
"Size check",
old_path,

46
tools/python/maps_generator/checks/default_check_set.py
View file

@ -8,7 +8,7 @@ from maps_generator.checks import check
from maps_generator.checks.check_addresses import get_addresses_check_set
from maps_generator.checks.check_categories import get_categories_check_set
from maps_generator.checks.check_log_levels import get_log_levels_check_set
from maps_generator.checks.check_mwm_types import get_mwm_all_types_check_set
from maps_generator.checks.check_mwm_types import get_all_mwm_types_check_set
from maps_generator.checks.check_mwm_types import get_mwm_type_check_set
from maps_generator.checks.check_sections import get_sections_existence_check_set
from maps_generator.checks.check_sections import get_sections_size_check_set
@ -37,30 +37,47 @@ def set_threshold(check_type_map: Mapping[CheckType, Threshold]):
_check_type_map = check_type_map
def get_threshold(check_type: CheckType) -> Threshold:
return _check_type_map[check_type]
def make_default_filter(check_type_map: Mapping[CheckType, Threshold] = None):
if check_type_map is None:
check_type_map = _check_type_map
def maker(check_type: CheckType):
threshold = check_type_map[check_type]
def make_default_filter(threshold: Threshold):
def default_filter(r: check.ResLine):
return check.norm(r.diff) > threshold.abs and check.get_rel(r) > threshold.rel
def default_filter(r: check.ResLine):
return (
check.norm(r.diff) > threshold.abs and check.get_rel(r) > threshold.rel
)
return default_filter
return default_filter
return maker
def get_mwm_check_sets_and_filters(
old_path: str, new_path: str, categories_path: str
) -> Mapping[check.Check, Callable]:
check_type_map_size = {
CheckType.low: Threshold(abs=20, rel=20000),
CheckType.medium: Threshold(abs=15, rel=15000),
CheckType.hard: Threshold(abs=10, rel=1000),
CheckType.strict: Threshold(abs=0, rel=0),
}
return {
get_categories_check_set(
old_path, new_path, categories_path
): make_default_filter,
): make_default_filter(),
get_mwm_type_check_set(
old_path, new_path, "sponsored-booking"
): make_default_filter,
get_mwm_all_types_check_set(old_path, new_path): make_default_filter,
get_size_check_set(old_path, new_path): make_default_filter,
get_sections_size_check_set(old_path, new_path): make_default_filter,
): make_default_filter(),
get_all_mwm_types_check_set(old_path, new_path): make_default_filter(),
get_size_check_set(old_path, new_path): make_default_filter(
check_type_map_size
),
get_sections_size_check_set(old_path, new_path): make_default_filter(
check_type_map_size
),
get_sections_existence_check_set(old_path, new_path): None,
}
@ -69,7 +86,7 @@ def get_logs_check_sets_and_filters(
old_path: str, new_path: str
) -> Mapping[check.Check, Callable]:
return {
get_addresses_check_set(old_path, new_path): make_default_filter,
get_addresses_check_set(old_path, new_path): make_default_filter(),
get_log_levels_check_set(old_path, new_path): None,
}
@ -88,12 +105,11 @@ def run_checks_and_print_results(
silent_if_no_results: bool = True,
file=sys.stdout,
):
threshold = get_threshold(check_type)
for check, make_filt in checks.items():
check.check()
_print_header(file, check.name)
check.print(
silent_if_no_results=silent_if_no_results,
filt=None if make_filt is None else make_filt(threshold),
filt=None if make_filt is None else make_filt(check_type),
file=file,
)

29
tools/python/maps_generator/checks/logs/logs_reader.py
View file

@ -55,13 +55,13 @@ class Log:
self.name = self.path.stem
self.is_stage_log = False
self.is_country_log = False
self.is_mwm_log = False
try:
get_stage_type(self.name)
self.is_stage_log = True
except AttributeError:
if self.name in env.COUNTRIES_NAMES:
self.is_country_log = True
if self.name in env.COUNTRIES_NAMES or self.name in env.WORLDS_NAMES:
self.is_mwm_log = True
self.lines = self._parse_lines()
@ -88,7 +88,7 @@ class Log:
if line is not None:
lines.append(line)
else:
logger.warn(f"Line '{logline}' was not parsed.")
logger.warn(f"Line was not parsed: {logline}")
logline = ""
with self.path.open() as logfile:
@ -186,21 +186,22 @@ def split_into_stages(log: Log) -> List[LogStage]:
return log_stages
def _is_worse(lhs: LogStage, rhs: LogStage) -> bool:
if (lhs.duration is None) ^ (rhs.duration is None):
return lhs.duration is None
if len(rhs.lines) > len(lhs.lines):
return True
return rhs.duration > lhs.duration
def normalize_logs(llogs: List[LogStage]) -> List[LogStage]:
def is_better(lhs: LogStage, rhs: LogStage) -> bool:
if (lhs.duration is None) ^ (rhs.duration is None):
return lhs.duration is None
if len(rhs.lines) > len(lhs.lines):
return True
return rhs.duration > lhs.duration
normalized_logs = []
buckets = {}
for log in llogs:
if log.name in buckets:
if is_better(normalized_logs[buckets[log.name]], log):
if _is_worse(normalized_logs[buckets[log.name]], log):
normalized_logs[buckets[log.name]] = log
else:
normalized_logs.append(log)

8
tools/python/maps_generator/generator/stages.py
View file

@ -147,6 +147,8 @@ def outer_stage(stage: Type[Stage]) -> Type[Stage]:
logfile = os.path.join(env.paths.log_path, f"{name}.log")
log_handler = create_file_handler(logfile)
logger.addHandler(log_handler)
# This message is used as an anchor for parsing logs.
# See maps_generator/checks/logs/logs_reader.py STAGE_START_MSG_PATTERN
logger.info(f"Stage {name}: start ...")
t = time.time()
try:
@ -165,6 +167,8 @@ def outer_stage(stage: Type[Stage]) -> Type[Stage]:
method(obj, env, *args, **kwargs)
finally:
d = time.time() - t
# This message is used as an anchor for parsing logs.
# See maps_generator/checks/logs/logs_reader.py STAGE_FINISH_MSG_PATTERN
logger.info(f"Stage {name}: finished in " f"{str(datetime.timedelta(seconds=d))}")
logger.removeHandler(log_handler)
@ -222,11 +226,15 @@ def country_stage_log(stage: Type[Stage]) -> Type[Stage]:
countries_meta[country]["logger"] = create_file_logger(log_file)
_logger, log_handler = countries_meta[country]["logger"]
# This message is used as an anchor for parsing logs.
# See maps_generator/checks/logs/logs_reader.py STAGE_START_MSG_PATTERN
_logger.info(f"Stage {name}: start ...")
t = time.time()
env.set_subprocess_out(log_handler.stream, country)
method(obj, env, country, *args, logger=_logger, **kwargs)
d = time.time() - t
# This message is used as an anchor for parsing logs.
# See maps_generator/checks/logs/logs_reader.py STAGE_FINISH_MSG_PATTERN
_logger.info(
f"Stage {name}: finished in "
f"{str(datetime.timedelta(seconds=d))}"

2
tools/python/maps_generator/tests/test_logs_reader.py
View file

@ -26,7 +26,7 @@ class TestLogsReader(unittest.TestCase):
def test_read_logs(self):
self.assertTrue(self.log.name.startswith("Czech_Jihovychod_Jihomoravsky kraj"))
self.assertTrue(self.log.is_country_log)
self.assertTrue(self.log.is_mwm_log)
self.assertFalse(self.log.is_stage_log)
self.assertEqual(len(self.log.lines), 46)

7
tools/python/maps_generator/update_generation_order.py
View file

@ -7,7 +7,8 @@ from maps_generator.checks.logs import logs_reader
def get_args():
parser = argparse.ArgumentParser(
description="This script generates file with countries that are ordered by time."
description="This script generates file with countries that are "
"ordered by time needed to generate them."
)
parser.add_argument(
"--output", type=str, required=True, help="Path to output file.",
@ -30,12 +31,12 @@ def main():
with ThreadPool() as pool:
order = pool.map(
process_log,
(log for log in logs_reader.LogsReader(args.logs) if log.is_country_log),
(log for log in logs_reader.LogsReader(args.logs) if log.is_mwm_log),
)
order.sort(key=lambda v: v[1], reverse=True)
with open(args.output, "w") as out:
out.write("# Mwm name Generation time\n")
out.write("# Mwm name\tGeneration time\n")
out.writelines("{}\t{}\n".format(*line) for line in order)