def tokenize_params(params): for param in params: if '|' in param: yield [p.strip() for p in param.split(' |')] else: yield param.strip() def tokenize_line(line): line = line.split(';')[0].strip() key_params = line.split(' ', maxsplit=1) if len(key_params) == 1: return tuple(key_params) else: key, params = key_params params = [p.strip() for p in params.split(',')] return key, list(tokenize_params(params)) def tokenize_lines(lines, prefix=""): for line in filter(bool, lines): line = line.strip() if line.startswith(prefix): yield tokenize_line(line)