diff --git a/tools/python/maps_generator/checks/check.py b/tools/python/maps_generator/checks/check.py index 9674fb4d82..c0e17af652 100644 --- a/tools/python/maps_generator/checks/check.py +++ b/tools/python/maps_generator/checks/check.py @@ -48,8 +48,55 @@ def get_rel(r: ResLine) -> bool: class Check(ABC): + """ + Base class for any checks. + Usual flow: + + # Create check object. + check = AnyCheck("ExampleCheck") + # Do work. + check.check() + + # Get results and process them + raw_result = check.get_result() + process_result(raw_result) + + # or print result + check.print() + """ def __init__(self, name: str): self.name = name + + def print(self, silent_if_no_results=False, filt=None, file=sys.stdout): + s = self.formatted_string(silent_if_no_results, filt) + if s: + print(s, file=file) + + @abstractmethod + def check(self): + """ + Performs a logic of the check. + """ + pass + + @abstractmethod + def get_result(self) -> Any: + """ + Returns a raw result of the check. + """ + pass + + @abstractmethod + def formatted_string(self, silent_if_no_results=False, *args, **kwargs) -> str: + """ + Returns a formatted string of a raw result of the check. + """ + pass + + +class CompareCheckBase(Check, ABC): + def __init__(self, name: str): + super().__init__(name) self.op: Callable[ [Any, Any], Any ] = lambda previous, current: current - previous @@ -77,25 +124,8 @@ class Check(ABC): def set_filt(self, filt: Callable[[Any], bool]): self.filt = filt - def print(self, silent_if_no_results=False, filt=None, file=sys.stdout): - s = self.formatted_string(silent_if_no_results, filt) - if s: - print(s, file=file) - @abstractmethod - def check(self): - pass - - @abstractmethod - def get_result(self) -> Any: - pass - - @abstractmethod - def formatted_string(self, silent_if_no_results=False, *args, **kwargs) -> str: - pass - - -class CompareCheck(Check): +class CompareCheck(CompareCheckBase): def __init__( self, name: str, old: Any, new: Any, ): @@ -156,7 +186,7 @@ class CompareCheck(Check): ) -class CompareCheckSet(Check): +class CompareCheckSet(CompareCheckBase): def __init__(self, name: str): super().__init__(name) @@ -215,15 +245,15 @@ class CompareCheckSet(Check): lines = [] if no_results: - lines.append(f"{' ' * _offset}No results.") + lines.append(f"{' ' * (_offset + 2)}No results.") for c in checks: - s = c.formatted_string(silent_if_no_results, filt, _offset + 1) + s = c.formatted_string(silent_if_no_results, filt, _offset + 2) if s: - lines.append(f"{' ' * _offset + ' '}{s}") + lines.append(f"{' ' * (_offset + 2)}{s}") for s in sets: - s = s.formatted_string(silent_if_no_results, filt, _offset + 1) + s = s.formatted_string(silent_if_no_results, filt, _offset + 2) if s: lines.append(s) diff --git a/tools/python/maps_generator/checks/check_addresses.py b/tools/python/maps_generator/checks/check_addresses.py index 49d5f83eae..e922f82b4c 100644 --- a/tools/python/maps_generator/checks/check_addresses.py +++ b/tools/python/maps_generator/checks/check_addresses.py @@ -13,9 +13,13 @@ ADDR_PATTERN = re.compile( def get_addresses_check_set(old_path: str, new_path: str) -> check.CompareCheckSet: + """ + Returns an addresses check set, that checks a difference in 'matched_percent' + addresses of BuildAddressTable between old logs and new logs. + """ def do(path: str): log = logs_reader.Log(path) - if not log.is_country_log: + if not log.is_mwm_log: return None found = logs_reader.find_and_parse(log.lines, ADDR_PATTERN) diff --git a/tools/python/maps_generator/checks/check_categories.py b/tools/python/maps_generator/checks/check_categories.py index 9a4314da4c..19b64c4676 100644 --- a/tools/python/maps_generator/checks/check_categories.py +++ b/tools/python/maps_generator/checks/check_categories.py @@ -32,7 +32,11 @@ def parse_groups(path): def get_categories_check_set( old_path: str, new_path: str, categories_path: str ) -> check.CompareCheckSet: - cs = check.CompareCheckSet("Sections categories check") + """ + Returns a categories check set, that checks a difference in a number of + objects of categories(from categories.txt) between old mwms and new mwms. + """ + cs = check.CompareCheckSet("Categories check") def make_do(indexes): def do(path): diff --git a/tools/python/maps_generator/checks/check_log_levels.py b/tools/python/maps_generator/checks/check_log_levels.py index dfe836347d..10f13636ec 100644 --- a/tools/python/maps_generator/checks/check_log_levels.py +++ b/tools/python/maps_generator/checks/check_log_levels.py @@ -13,6 +13,11 @@ def _get_log_stages(path): def get_log_levels_check_set(old_path: str, new_path: str) -> check.CompareCheckSet: + """ + Returns a log levels check set, that checks a difference in a number of + message levels from warning and higher for each stage between old mwms + and new mwms. + """ cs = check.CompareCheckSet("Log levels check") def make_do(level, stage_name, cache={}): diff --git a/tools/python/maps_generator/checks/check_mwm_types.py b/tools/python/maps_generator/checks/check_mwm_types.py index f9cd9ca0f2..4af1a34b32 100644 --- a/tools/python/maps_generator/checks/check_mwm_types.py +++ b/tools/python/maps_generator/checks/check_mwm_types.py @@ -19,23 +19,31 @@ def count_all_types(path: str): def get_mwm_type_check_set( - old_path: str, new_path: str, _type: Union[str, int] + old_path: str, new_path: str, type_: Union[str, int] ) -> check.CompareCheckSet: - if isinstance(_type, str): - _type = type_index(_type) - assert _type >= 0, _type + """ + Returns a mwm type check set, that checks a difference in a number of + type [type_] between old mwms and new mwms. + """ + if isinstance(type_, str): + type_ = type_index(type_) + assert type_ >= 0, type_ return check.build_check_set_for_files( - f"Types check [{readable_type(_type)}]", + f"Types check [{readable_type(type_)}]", old_path, new_path, ext=".mwm", - do=lambda path: count_all_types(path)[_type], + do=lambda path: count_all_types(path)[type_], ) -def get_mwm_all_types_check_set(old_path: str, new_path: str) -> check.CompareCheckSet: - cs = check.CompareCheckSet("Sections categories check") +def get_all_mwm_types_check_set(old_path: str, new_path: str) -> check.CompareCheckSet: + """ + Returns a all mwm types check set, that checks a difference in a number of + each type between old mwms and new mwms. + """ + cs = check.CompareCheckSet("Mwm types check") def make_do(index): return lambda path: count_all_types(path)[index] diff --git a/tools/python/maps_generator/checks/check_sections.py b/tools/python/maps_generator/checks/check_sections.py index e708b172e7..244912213e 100644 --- a/tools/python/maps_generator/checks/check_sections.py +++ b/tools/python/maps_generator/checks/check_sections.py @@ -10,7 +10,9 @@ class SectionNames: self.sections = sections def __sub__(self, other): - return SectionNames(self.sections - other.sections) + return SectionNames( + {k: self.sections[k] for k in set(self.sections) - set(other.sections)} + ) def __lt__(self, other): if isinstance(other, int): @@ -46,8 +48,8 @@ def get_appeared_sections_check_set( old_path, new_path, ext=".mwm", - do=lambda path: SectionNames(read_sections(path).keys()), - diff_format=lambda s: ",".join(s.sections), + do=lambda path: SectionNames(read_sections(path)), + diff_format=lambda s: ", ".join(f"{k}:{v.size}" for k, v in s.sections.items()), format=lambda s: f"number of sections: {len(s.sections)}", ) @@ -60,9 +62,9 @@ def get_disappeared_sections_check_set( old_path, new_path, ext=".mwm", - do=lambda path: SectionNames(read_sections(path).keys()), + do=lambda path: SectionNames(read_sections(path)), op=lambda previous, current: previous - current, - diff_format=lambda s: ",".join(s.sections), + diff_format=lambda s: ", ".join(f"{k}:{v.size}" for k, v in s.sections.items()), format=lambda s: f"number of sections: {len(s.sections)}", ) @@ -70,6 +72,10 @@ def get_disappeared_sections_check_set( def get_sections_existence_check_set( old_path: str, new_path: str ) -> check.CompareCheckSet: + """ + Returns a sections existence check set, that checks appeared and + disappeared sections between old mwms and new mwms. + """ cs = check.CompareCheckSet("Sections existence check") cs.add_check(get_appeared_sections_check_set(old_path, new_path)) cs.add_check(get_disappeared_sections_check_set(old_path, new_path)) @@ -86,6 +92,10 @@ def _get_sections_set(path): def get_sections_size_check_set(old_path: str, new_path: str) -> check.CompareCheckSet: + """ + Returns a sections size check set, that checks a difference in a size + of each sections of mwm between old mwms and new mwms. + """ sections_set = _get_sections_set(old_path) sections_set.update(_get_sections_set(new_path)) diff --git a/tools/python/maps_generator/checks/check_size.py b/tools/python/maps_generator/checks/check_size.py index 56e6b124e6..38178db668 100644 --- a/tools/python/maps_generator/checks/check_size.py +++ b/tools/python/maps_generator/checks/check_size.py @@ -4,6 +4,10 @@ from maps_generator.checks import check def get_size_check_set(old_path: str, new_path: str) -> check.CompareCheckSet: + """ + Returns a size check set, that checks a difference in a size of mwm between + old mwms and new mwms. + """ return check.build_check_set_for_files( "Size check", old_path, diff --git a/tools/python/maps_generator/checks/default_check_set.py b/tools/python/maps_generator/checks/default_check_set.py index 3c610f0b0e..fe4600e868 100644 --- a/tools/python/maps_generator/checks/default_check_set.py +++ b/tools/python/maps_generator/checks/default_check_set.py @@ -8,7 +8,7 @@ from maps_generator.checks import check from maps_generator.checks.check_addresses import get_addresses_check_set from maps_generator.checks.check_categories import get_categories_check_set from maps_generator.checks.check_log_levels import get_log_levels_check_set -from maps_generator.checks.check_mwm_types import get_mwm_all_types_check_set +from maps_generator.checks.check_mwm_types import get_all_mwm_types_check_set from maps_generator.checks.check_mwm_types import get_mwm_type_check_set from maps_generator.checks.check_sections import get_sections_existence_check_set from maps_generator.checks.check_sections import get_sections_size_check_set @@ -37,30 +37,47 @@ def set_threshold(check_type_map: Mapping[CheckType, Threshold]): _check_type_map = check_type_map -def get_threshold(check_type: CheckType) -> Threshold: - return _check_type_map[check_type] +def make_default_filter(check_type_map: Mapping[CheckType, Threshold] = None): + if check_type_map is None: + check_type_map = _check_type_map + def maker(check_type: CheckType): + threshold = check_type_map[check_type] -def make_default_filter(threshold: Threshold): - def default_filter(r: check.ResLine): - return check.norm(r.diff) > threshold.abs and check.get_rel(r) > threshold.rel + def default_filter(r: check.ResLine): + return ( + check.norm(r.diff) > threshold.abs and check.get_rel(r) > threshold.rel + ) - return default_filter + return default_filter + + return maker def get_mwm_check_sets_and_filters( old_path: str, new_path: str, categories_path: str ) -> Mapping[check.Check, Callable]: + check_type_map_size = { + CheckType.low: Threshold(abs=20, rel=20000), + CheckType.medium: Threshold(abs=15, rel=15000), + CheckType.hard: Threshold(abs=10, rel=1000), + CheckType.strict: Threshold(abs=0, rel=0), + } + return { get_categories_check_set( old_path, new_path, categories_path - ): make_default_filter, + ): make_default_filter(), get_mwm_type_check_set( old_path, new_path, "sponsored-booking" - ): make_default_filter, - get_mwm_all_types_check_set(old_path, new_path): make_default_filter, - get_size_check_set(old_path, new_path): make_default_filter, - get_sections_size_check_set(old_path, new_path): make_default_filter, + ): make_default_filter(), + get_all_mwm_types_check_set(old_path, new_path): make_default_filter(), + get_size_check_set(old_path, new_path): make_default_filter( + check_type_map_size + ), + get_sections_size_check_set(old_path, new_path): make_default_filter( + check_type_map_size + ), get_sections_existence_check_set(old_path, new_path): None, } @@ -69,7 +86,7 @@ def get_logs_check_sets_and_filters( old_path: str, new_path: str ) -> Mapping[check.Check, Callable]: return { - get_addresses_check_set(old_path, new_path): make_default_filter, + get_addresses_check_set(old_path, new_path): make_default_filter(), get_log_levels_check_set(old_path, new_path): None, } @@ -88,12 +105,11 @@ def run_checks_and_print_results( silent_if_no_results: bool = True, file=sys.stdout, ): - threshold = get_threshold(check_type) for check, make_filt in checks.items(): check.check() _print_header(file, check.name) check.print( silent_if_no_results=silent_if_no_results, - filt=None if make_filt is None else make_filt(threshold), + filt=None if make_filt is None else make_filt(check_type), file=file, ) diff --git a/tools/python/maps_generator/checks/logs/logs_reader.py b/tools/python/maps_generator/checks/logs/logs_reader.py index a9ddbda488..1771527262 100644 --- a/tools/python/maps_generator/checks/logs/logs_reader.py +++ b/tools/python/maps_generator/checks/logs/logs_reader.py @@ -55,13 +55,13 @@ class Log: self.name = self.path.stem self.is_stage_log = False - self.is_country_log = False + self.is_mwm_log = False try: get_stage_type(self.name) self.is_stage_log = True except AttributeError: - if self.name in env.COUNTRIES_NAMES: - self.is_country_log = True + if self.name in env.COUNTRIES_NAMES or self.name in env.WORLDS_NAMES: + self.is_mwm_log = True self.lines = self._parse_lines() @@ -88,7 +88,7 @@ class Log: if line is not None: lines.append(line) else: - logger.warn(f"Line '{logline}' was not parsed.") + logger.warn(f"Line was not parsed: {logline}") logline = "" with self.path.open() as logfile: @@ -186,21 +186,22 @@ def split_into_stages(log: Log) -> List[LogStage]: return log_stages +def _is_worse(lhs: LogStage, rhs: LogStage) -> bool: + if (lhs.duration is None) ^ (rhs.duration is None): + return lhs.duration is None + + if len(rhs.lines) > len(lhs.lines): + return True + + return rhs.duration > lhs.duration + + def normalize_logs(llogs: List[LogStage]) -> List[LogStage]: - def is_better(lhs: LogStage, rhs: LogStage) -> bool: - if (lhs.duration is None) ^ (rhs.duration is None): - return lhs.duration is None - - if len(rhs.lines) > len(lhs.lines): - return True - - return rhs.duration > lhs.duration - normalized_logs = [] buckets = {} for log in llogs: if log.name in buckets: - if is_better(normalized_logs[buckets[log.name]], log): + if _is_worse(normalized_logs[buckets[log.name]], log): normalized_logs[buckets[log.name]] = log else: normalized_logs.append(log) diff --git a/tools/python/maps_generator/generator/stages.py b/tools/python/maps_generator/generator/stages.py index d970df90da..bdfd18d6c2 100644 --- a/tools/python/maps_generator/generator/stages.py +++ b/tools/python/maps_generator/generator/stages.py @@ -147,6 +147,8 @@ def outer_stage(stage: Type[Stage]) -> Type[Stage]: logfile = os.path.join(env.paths.log_path, f"{name}.log") log_handler = create_file_handler(logfile) logger.addHandler(log_handler) + # This message is used as an anchor for parsing logs. + # See maps_generator/checks/logs/logs_reader.py STAGE_START_MSG_PATTERN logger.info(f"Stage {name}: start ...") t = time.time() try: @@ -165,6 +167,8 @@ def outer_stage(stage: Type[Stage]) -> Type[Stage]: method(obj, env, *args, **kwargs) finally: d = time.time() - t + # This message is used as an anchor for parsing logs. + # See maps_generator/checks/logs/logs_reader.py STAGE_FINISH_MSG_PATTERN logger.info(f"Stage {name}: finished in " f"{str(datetime.timedelta(seconds=d))}") logger.removeHandler(log_handler) @@ -222,11 +226,15 @@ def country_stage_log(stage: Type[Stage]) -> Type[Stage]: countries_meta[country]["logger"] = create_file_logger(log_file) _logger, log_handler = countries_meta[country]["logger"] + # This message is used as an anchor for parsing logs. + # See maps_generator/checks/logs/logs_reader.py STAGE_START_MSG_PATTERN _logger.info(f"Stage {name}: start ...") t = time.time() env.set_subprocess_out(log_handler.stream, country) method(obj, env, country, *args, logger=_logger, **kwargs) d = time.time() - t + # This message is used as an anchor for parsing logs. + # See maps_generator/checks/logs/logs_reader.py STAGE_FINISH_MSG_PATTERN _logger.info( f"Stage {name}: finished in " f"{str(datetime.timedelta(seconds=d))}" diff --git a/tools/python/maps_generator/tests/test_logs_reader.py b/tools/python/maps_generator/tests/test_logs_reader.py index 57a9234e0a..2ecb513220 100644 --- a/tools/python/maps_generator/tests/test_logs_reader.py +++ b/tools/python/maps_generator/tests/test_logs_reader.py @@ -26,7 +26,7 @@ class TestLogsReader(unittest.TestCase): def test_read_logs(self): self.assertTrue(self.log.name.startswith("Czech_Jihovychod_Jihomoravsky kraj")) - self.assertTrue(self.log.is_country_log) + self.assertTrue(self.log.is_mwm_log) self.assertFalse(self.log.is_stage_log) self.assertEqual(len(self.log.lines), 46) diff --git a/tools/python/maps_generator/update_generation_order.py b/tools/python/maps_generator/update_generation_order.py index 9ff97f79d8..6d144d298c 100644 --- a/tools/python/maps_generator/update_generation_order.py +++ b/tools/python/maps_generator/update_generation_order.py @@ -7,7 +7,8 @@ from maps_generator.checks.logs import logs_reader def get_args(): parser = argparse.ArgumentParser( - description="This script generates file with countries that are ordered by time." + description="This script generates file with countries that are " + "ordered by time needed to generate them." ) parser.add_argument( "--output", type=str, required=True, help="Path to output file.", @@ -30,12 +31,12 @@ def main(): with ThreadPool() as pool: order = pool.map( process_log, - (log for log in logs_reader.LogsReader(args.logs) if log.is_country_log), + (log for log in logs_reader.LogsReader(args.logs) if log.is_mwm_log), ) order.sort(key=lambda v: v[1], reverse=True) with open(args.output, "w") as out: - out.write("# Mwm name Generation time\n") + out.write("# Mwm name\tGeneration time\n") out.writelines("{}\t{}\n".format(*line) for line in order)