# mypy: ignore-errors

import json
import os
import sys
from io import BytesIO, StringIO
from unittest import mock

import pytest

from .. import metadata, manifestupdate, wptcommandline, wptmanifest
from ..update.update import WPTUpdate
from ..update.base import StepRunner, Step
from mozlog import structuredlog, handlers, formatters

here = os.path.dirname(__file__)
sys.path.insert(0, os.path.join(here, os.pardir, os.pardir, os.pardir))
from manifest import manifest, item as manifest_item, utils


def rel_path_to_test_url(rel_path):
    assert not os.path.isabs(rel_path)
    return rel_path.replace(os.sep, "/")


def SourceFileWithTest(path, hash, cls, *args):
    path_parts = tuple(path.split("/"))
    path = utils.to_os_path(path)
    s = mock.Mock(rel_path=path, rel_path_parts=path_parts, hash=hash)
    test = cls("/foobar", path, "/", rel_path_to_test_url(path), *args)
    s.manifest_items = mock.Mock(return_value=(cls.item_type, [test]))
    return s


def tree_and_sourcefile_mocks(source_files):
    paths_dict = {}
    tree = []
    for source_file, file_hash, updated in source_files:
        paths_dict[source_file.rel_path] = source_file
        tree.append([source_file.rel_path, file_hash, updated])

    def MockSourceFile(tests_root, path, url_base, file_hash):
        return paths_dict[path]

    return tree, MockSourceFile


item_classes = {"testharness": manifest_item.TestharnessTest,
                "reftest": manifest_item.RefTest,
                "manual": manifest_item.ManualTest,
                "wdspec": manifest_item.WebDriverSpecTest,
                "conformancechecker": manifest_item.ConformanceCheckerTest,
                "visual": manifest_item.VisualTest,
                "support": manifest_item.SupportFile}


default_run_info = {"debug": False, "os": "linux", "version": "18.04", "processor": "x86_64", "bits": 64}
test_id = "/path/to/test.htm"
dir_id = "path/to/__dir__"


def reset_globals():
    metadata.prop_intern.clear()
    metadata.run_info_intern.clear()
    metadata.status_intern.clear()


def get_run_info(overrides):
    run_info = default_run_info.copy()
    run_info.update(overrides)
    return run_info


def update(tests, *logs, **kwargs):
    full_update = kwargs.pop("full_update", False)
    disable_intermittent = kwargs.pop("disable_intermittent", False)
    update_intermittent = kwargs.pop("update_intermittent", False)
    remove_intermittent = kwargs.pop("remove_intermittent", False)
    assert not kwargs
    id_test_map, updater = create_updater(tests)

    for log in logs:
        log = create_log(log)
        updater.update_from_log(log)

    update_properties = (["debug", "os", "version", "processor"],
                         {"os": ["version"], "processor": ["bits"]})

    expected_data = {}
    metadata.load_expected = lambda _, __, test_path, *args: expected_data.get(test_path)
    for test_path, test_ids, test_type, manifest_str in tests:
        test_path = utils.to_os_path(test_path)
        expected_data[test_path] = manifestupdate.compile(BytesIO(manifest_str),
                                                          test_path,
                                                          "/",
                                                          update_properties,
                                                          update_intermittent,
                                                          remove_intermittent)

    return list(metadata.update_results(id_test_map,
                                        update_properties,
                                        full_update,
                                        disable_intermittent,
                                        update_intermittent,
                                        remove_intermittent))


def create_updater(tests, url_base="/", **kwargs):
    id_test_map = {}
    m = create_test_manifest(tests, url_base)

    reset_globals()
    id_test_map = metadata.create_test_tree(None, m)

    return id_test_map, metadata.ExpectedUpdater(id_test_map, **kwargs)


def create_log(entries):
    data = StringIO()
    if isinstance(entries, list):
        logger = structuredlog.StructuredLogger("expected_test")
        handler = handlers.StreamHandler(data, formatters.JSONFormatter())
        logger.add_handler(handler)

        for item in entries:
            action, kwargs = item
            getattr(logger, action)(**kwargs)
        logger.remove_handler(handler)
    else:
        data.write(json.dumps(entries))
    data.seek(0)
    return data


def suite_log(entries, run_info=None):
    _run_info = default_run_info.copy()
    if run_info:
        _run_info.update(run_info)
    return ([("suite_start", {"tests": [], "run_info": _run_info})] +
            entries +
            [("suite_end", {})])


def create_test_manifest(tests, url_base="/"):
    source_files = []
    for i, (test, _, test_type, _) in enumerate(tests):
        if test_type:
            source_files.append(SourceFileWithTest(test, str(i) * 40, item_classes[test_type]))
    m = manifest.Manifest("")
    tree, sourcefile_mock = tree_and_sourcefile_mocks((item, None, True) for item in source_files)
    with mock.patch("manifest.manifest.SourceFile", side_effect=sourcefile_mock):
        m.update(tree)
    return m


def test_update_0():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log = suite_log([("test_start", {"test": "/path/to/test.htm"}),
                     ("test_status", {"test": "/path/to/test.htm",
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "FAIL"}),
                     ("test_end", {"test": "/path/to/test.htm",
                                   "status": "OK"})])

    updated = update(tests, log)

    assert len(updated) == 1
    assert updated[0][1].is_empty


def test_update_1():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: ERROR""")]

    log = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "ERROR"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log)

    new_manifest = updated[0][1]
    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get("expected", default_run_info) == "FAIL"


def test_update_known_intermittent_1():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: PASS""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "PASS"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "PASS"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_2 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "PASS"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, log_1, log_2, update_intermittent=True)

    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == ["PASS", "FAIL"]


def test_update_known_intermittent_2():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: PASS""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "PASS"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, update_intermittent=True)

    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == "FAIL"


def test_update_existing_known_intermittent():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: [PASS, FAIL]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "ERROR",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_2 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, log_1, log_2, update_intermittent=True)

    new_manifest = updated[0][1]
    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == ["PASS", "ERROR", "FAIL"]


def test_update_remove_previous_intermittent():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: [PASS, FAIL]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "ERROR",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_2 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests,
                     log_0,
                     log_1,
                     log_2,
                     update_intermittent=True,
                     remove_intermittent=True)

    new_manifest = updated[0][1]
    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == ["PASS", "ERROR"]


def test_update_new_test_with_intermittent():
    tests = [("path/to/test.htm", [test_id], "testharness", None)]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    updated = update(tests, log_0, log_1, log_2, update_intermittent=True)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test("test.htm") is None
    assert len(new_manifest.get_test(test_id).children) == 1
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == ["PASS", "FAIL"]


def test_update_expected_tie_resolution():
    tests = [("path/to/test.htm", [test_id], "testharness", None)]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    updated = update(tests, log_0, log_1, update_intermittent=True)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == ["PASS", "FAIL"]


def test_update_no_reorder_expected():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: [PASS, FAIL]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    updated = update(tests, log_0, log_1, log_2, update_intermittent=True)
    assert not updated


def test_update_and_preserve_unchanged_expected_intermittent():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected:
    if os == "android": [PASS, FAIL]
    FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "FAIL",
                                     "expected": "PASS",
                                     "known_intermittent": ["FAIL"]})],
                      run_info={"os": "android"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS",
                                     "expected": "PASS",
                                     "known_intermittent": ["FAIL"]})],
                      run_info={"os": "android"})

    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS",
                                     "expected": "FAIL"})])

    updated = update(tests, log_0, log_1, log_2)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified

    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "android"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == ["PASS", "FAIL"]
    assert new_manifest.get_test(test_id).get(
        "expected", default_run_info) == "PASS"


def test_update_intermittent():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected:
      if os == "linux" or os == "android": [PASS, FAIL, ERROR]""")]

    # Logs where the test requires an update, but we don't want to update the
    # intermittent status
    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL", "ERROR"]}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL", "ERROR"]}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL", "ERROR"]}),
                       ("test_end", {"test": test_id,
                                     "status": "TIMEOUT"})],
                      run_info={"os": "android"})

    updated = update(tests, log_0, log_1, log_2, update_intermittent=True)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified

    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "android"})
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == "TIMEOUT"

    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == ["PASS", "FAIL", "ERROR"]


def test_update_test_with_intermittent_to_one_expected_status():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: [PASS, FAIL]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "ERROR",
                                      "expected": "PASS",
                                      "known_intermittent": ["FAIL"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0)

    new_manifest = updated[0][1]
    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == "ERROR"


def test_update_intermittent_with_conditions():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected:
    if os == "android": [PASS, FAIL]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "TIMEOUT",
                                     "expected": "PASS",
                                     "known_intermittent": ["FAIL"]})],
                      run_info={"os": "android"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS",
                                     "expected": "PASS",
                                     "known_intermittent": ["FAIL"]})],
                      run_info={"os": "android"})

    updated = update(tests, log_0, log_1, update_intermittent=True)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified

    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "android"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == ["PASS", "TIMEOUT", "FAIL"]


def test_update_and_remove_intermittent_with_conditions():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected:
    if os == "android": [PASS, FAIL]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "TIMEOUT",
                                     "expected": "PASS",
                                     "known_intermittent": ["FAIL"]})],
                      run_info={"os": "android"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS",
                                     "expected": "PASS",
                                     "known_intermittent": ["FAIL"]})],
                      run_info={"os": "android"})

    updated = update(tests, log_0, log_1, update_intermittent=True, remove_intermittent=True)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified

    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "android"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == ["PASS", "TIMEOUT"]


def test_update_intermittent_full():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected:
      if os == "mac": [FAIL, TIMEOUT]
      FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "FAIL",
                                      "known_intermittent": ["TIMEOUT"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})],
                     run_info={"os": "mac"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, log_1, update_intermittent=True, full_update=True)

    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "mac"})
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == ["FAIL", "TIMEOUT"]
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == "FAIL"


def test_update_intermittent_full_remove():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected:
      if os == "mac": [FAIL, TIMEOUT, PASS]
      FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "FAIL",
                                      "known_intermittent": ["TIMEOUT", "PASS"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})],
                     run_info={"os": "mac"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "TIMEOUT",
                                      "expected": "FAIL",
                                      "known_intermittent": ["TIMEOUT", "PASS"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})],
                     run_info={"os": "mac"})

    log_2 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, log_1, log_2, update_intermittent=True,
                     full_update=True, remove_intermittent=True)

    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "mac"})
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == ["FAIL", "TIMEOUT"]
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == "FAIL"


def test_full_update():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected:
      if os == "mac": [FAIL, TIMEOUT]
      FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "FAIL",
                                      "known_intermittent": ["TIMEOUT"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})],
                     run_info={"os": "mac"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, log_1, full_update=True)

    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "mac"})
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == "FAIL"


def test_full_orphan():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: FAIL
    [subsub test]
      expected: TIMEOUT
  [test2]
    expected: ERROR
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "FAIL"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])


    updated = update(tests, log_0, full_update=True)

    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert len(new_manifest.get_test(test_id).children[0].children) == 0
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", default_run_info) == "FAIL"
    assert len(new_manifest.get_test(test_id).children) == 1


def test_update_no_reorder_expected_full_conditions():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected:
      if os == "mac": [FAIL, TIMEOUT]
      [FAIL, PASS]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "TIMEOUT",
                                      "expected": "FAIL",
                                      "known_intermittent": ["TIMEOUT"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})],
                     run_info={"os": "mac"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "TIMEOUT",
                                      "expected": "FAIL",
                                      "known_intermittent": ["TIMEOUT"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})],
                     run_info={"os": "mac"})

    log_2 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "FAIL",
                                      "known_intermittent": ["PASS"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    log_3 = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "PASS",
                                      "expected": "FAIL",
                                      "known_intermittent": ["PASS"]}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log_0, log_1, log_2, log_3, update_intermittent=True, full_update=True)
    assert not updated


def test_skip_0():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "FAIL"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])

    updated = update(tests, log)
    assert not updated


def test_new_subtest():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "expected": "FAIL"}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test2",
                                      "status": "FAIL",
                                      "expected": "PASS"}),
                     ("test_end", {"test": test_id,
                                   "status": "OK"})])
    updated = update(tests, log)
    new_manifest = updated[0][1]
    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get("expected", default_run_info) == "FAIL"
    assert new_manifest.get_test(test_id).children[1].get("expected", default_run_info) == "FAIL"


def test_update_subtest():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  expected:
    if os == "linux": [OK, ERROR]
  [test1]
    expected: FAIL""")]

    log = suite_log([("test_start", {"test": test_id}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test1",
                                      "status": "FAIL",
                                      "known_intermittent": []}),
                     ("test_status", {"test": test_id,
                                      "subtest": "test2",
                                      "status": "FAIL",
                                      "expected": "PASS",
                                      "known_intermittent": []}),
                     ("test_end", {"test": test_id,
                                   "status": "OK",
                                   "known_intermittent": ["ERROR"]})])
    updated = update(tests, log)
    new_manifest = updated[0][1]
    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get("expected", default_run_info) == "FAIL"


def test_update_multiple_0():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "os": "osx"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "TIMEOUT",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "os": "linux"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    run_info_1 = default_run_info.copy()
    run_info_1.update({"debug": False, "os": "osx"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"debug": False, "os": "linux"})
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", {"debug": False, "os": "linux"}) == "TIMEOUT"


def test_update_multiple_1():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "osx"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "TIMEOUT",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "linux"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "osx"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"os": "linux"})
    run_info_3 = default_run_info.copy()
    run_info_3.update({"os": "win"})

    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_2) == "TIMEOUT"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_3) == "FAIL"


def test_update_multiple_2():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "os": "osx"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "TIMEOUT",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": True, "os": "osx"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    run_info_1 = default_run_info.copy()
    run_info_1.update({"debug": False, "os": "osx"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"debug": True, "os": "osx"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_2) == "TIMEOUT"


def test_update_multiple_3():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected:
      if debug: FAIL
      if not debug and os == "osx": TIMEOUT""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "os": "osx"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "TIMEOUT",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": True, "os": "osx"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    run_info_1 = default_run_info.copy()
    run_info_1.update({"debug": False, "os": "osx"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"debug": True, "os": "osx"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_2) == "TIMEOUT"


def test_update_ignore_existing():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected:
      if debug: TIMEOUT
      if not debug and os == "osx": NOTRUN""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "os": "linux"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": True, "os": "windows"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    run_info_1 = default_run_info.copy()
    run_info_1.update({"debug": False, "os": "linux"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"debug": False, "os": "osx"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_2) == "NOTRUN"


def test_update_new_test():
    tests = [("path/to/test.htm", [test_id], "testharness", None)]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])
    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    run_info_1 = default_run_info.copy()

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test("test.htm") is None
    assert len(new_manifest.get_test(test_id).children) == 1
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"


def test_update_duplicate():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected: ERROR""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS"})])
    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "FAIL"})])

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]
    run_info_1 = default_run_info.copy()

    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == "ERROR"


def test_update_disable_intermittent():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected: ERROR""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS"})])
    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "FAIL"})])

    updated = update(tests, log_0, log_1, disable_intermittent="Some message")
    new_manifest = updated[0][1]
    run_info_1 = default_run_info.copy()

    assert new_manifest.get_test(test_id).get(
        "disabled", run_info_1) == "Some message"


def test_update_stability_conditional_instability():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected: ERROR""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "PASS"})],
                      run_info={"os": "linux"})
    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "FAIL"})],
                      run_info={"os": "linux"})
    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "status": "FAIL"})],
                      run_info={"os": "mac"})

    updated = update(tests, log_0, log_1, log_2, disable_intermittent="Some message")
    new_manifest = updated[0][1]
    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "linux"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"os": "mac"})

    assert new_manifest.get_test(test_id).get(
        "disabled", run_info_1) == "Some message"
    with pytest.raises(KeyError):
        assert new_manifest.get_test(test_id).get(
            "disabled", run_info_2)
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_2) == "FAIL"


def test_update_full():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected:
      if debug: TIMEOUT
      if not debug and os == "osx": NOTRUN

  [test2]
    expected: FAIL

[test.js]
  [test1]
    expected: FAIL
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "ERROR",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": True})

    updated = update(tests, log_0, log_1, full_update=True)
    new_manifest = updated[0][1]

    run_info_1 = default_run_info.copy()
    run_info_1.update({"debug": False, "os": "win"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"debug": True, "os": "osx"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test("test.js") is None
    assert len(new_manifest.get_test(test_id).children) == 1
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_2) == "ERROR"


def test_update_full_unknown():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected:
      if release_or_beta: ERROR
      if not debug and os == "osx": NOTRUN
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "release_or_beta": False})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": True, "release_or_beta": False})

    updated = update(tests, log_0, log_1, full_update=True)
    new_manifest = updated[0][1]

    run_info_1 = default_run_info.copy()
    run_info_1.update({"release_or_beta": False})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"release_or_beta": True})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_1) == "FAIL"
    assert new_manifest.get_test(test_id).children[0].get(
        "expected", run_info_2) == "ERROR"


def test_update_full_unknown_missing():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [subtest_deleted]
    expected:
      if release_or_beta: ERROR
      FAIL
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "PASS"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"debug": False, "release_or_beta": False})

    updated = update(tests, log_0, full_update=True)
    assert len(updated) == 0


def test_update_default():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  [test1]
    expected:
      if os == "mac": FAIL
      ERROR""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "FAIL"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "mac"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "expected": "ERROR"}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "linux"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    assert new_manifest.is_empty
    assert new_manifest.modified


def test_update_default_1():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected:
    if os == "mac": TIMEOUT
    ERROR""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "FAIL"})],
                      run_info={"os": "linux"})

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified

    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "mac"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"os": "win"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == "TIMEOUT"
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_2) == "FAIL"


def test_update_default_2():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""
[test.htm]
  expected:
    if os == "mac": TIMEOUT
    ERROR""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "TIMEOUT"})],
                      run_info={"os": "linux"})

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified

    run_info_1 = default_run_info.copy()
    run_info_1.update({"os": "mac"})
    run_info_2 = default_run_info.copy()
    run_info_2.update({"os": "win"})

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_1) == "TIMEOUT"
    assert new_manifest.get_test(test_id).get(
        "expected", run_info_2) == "TIMEOUT"


def test_update_assertion_count_0():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  max-asserts: 4
  min-asserts: 2
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 6,
                                            "min_expected": 2,
                                            "max_expected": 4}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get("max-asserts") == "7"
    assert new_manifest.get_test(test_id).get("min-asserts") == "2"


def test_update_assertion_count_1():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  max-asserts: 4
  min-asserts: 2
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 1,
                                            "min_expected": 2,
                                            "max_expected": 4}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get("max-asserts") == "4"
    assert new_manifest.get_test(test_id).has_key("min-asserts") is False


def test_update_assertion_count_2():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  max-asserts: 4
  min-asserts: 2
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 3,
                                            "min_expected": 2,
                                            "max_expected": 4}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})])

    updated = update(tests, log_0)
    assert not updated


def test_update_assertion_count_3():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]
  max-asserts: 4
  min-asserts: 2
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 6,
                                            "min_expected": 2,
                                            "max_expected": 4}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "windows"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 7,
                                            "min_expected": 2,
                                            "max_expected": 4}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "linux"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get("max-asserts") == "8"
    assert new_manifest.get_test(test_id).get("min-asserts") == "2"


def test_update_assertion_count_4():
    tests = [("path/to/test.htm", [test_id], "testharness", b"""[test.htm]""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 6,
                                            "min_expected": 0,
                                            "max_expected": 0}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "windows"})

    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("assertion_count", {"test": test_id,
                                            "count": 7,
                                            "min_expected": 0,
                                            "max_expected": 0}),
                       ("test_end", {"test": test_id,
                                     "status": "OK"})],
                      run_info={"os": "linux"})

    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get_test(test_id).get("max-asserts") == "8"
    assert new_manifest.get_test(test_id).has_key("min-asserts") is False


def test_update_lsan_0():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"")]

    log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
                                      "frames": ["foo", "bar"]})])


    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get("lsan-allowed") == ["foo"]


def test_update_lsan_1():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"""
lsan-allowed: [foo]""")]

    log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
                                      "frames": ["foo", "bar"]}),
                       ("lsan_leak", {"scope": "path/to/",
                                      "frames": ["baz", "foobar"]})])


    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get("lsan-allowed") == ["baz", "foo"]


def test_update_lsan_2():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/__dir__", ["path/__dir__"], None, b"""
lsan-allowed: [foo]"""),
             ("path/to/__dir__", [dir_id], None, b"")]

    log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
                                      "frames": ["foo", "bar"],
                                      "allowed_match": ["foo"]}),
                       ("lsan_leak", {"scope": "path/to/",
                                      "frames": ["baz", "foobar"]})])


    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get("lsan-allowed") == ["baz"]


def test_update_lsan_3():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"")]

    log_0 = suite_log([("lsan_leak", {"scope": "path/to/",
                                      "frames": ["foo", "bar"]})],
                      run_info={"os": "win"})

    log_1 = suite_log([("lsan_leak", {"scope": "path/to/",
                                      "frames": ["baz", "foobar"]})],
                      run_info={"os": "linux"})


    updated = update(tests, log_0, log_1)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get("lsan-allowed") == ["baz", "foo"]


def test_update_wptreport_0():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  [test1]
    expected: FAIL""")]

    log = {"run_info": default_run_info.copy(),
           "results": [
               {"test": "/path/to/test.htm",
                "subtests": [{"name": "test1",
                              "status": "PASS",
                              "expected": "FAIL"}],
                "status": "OK"}]}

    updated = update(tests, log)

    assert len(updated) == 1
    assert updated[0][1].is_empty


def test_update_wptreport_1():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"")]

    log = {"run_info": default_run_info.copy(),
           "results": [],
           "lsan_leaks": [{"scope": "path/to/",
                           "frames": ["baz", "foobar"]}]}

    updated = update(tests, log)

    assert len(updated) == 1
    assert updated[0][1].get("lsan-allowed") == ["baz"]


def test_update_leak_total_0():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"")]

    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
                                          "process": "default",
                                          "bytes": 100,
                                          "threshold": 0,
                                          "objects": []})])

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get("leak-threshold") == ['default:51200']


def test_update_leak_total_1():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"")]

    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
                                          "process": "default",
                                          "bytes": 100,
                                          "threshold": 1000,
                                          "objects": []})])

    updated = update(tests, log_0)
    assert not updated


def test_update_leak_total_2():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"""
leak-total: 110""")]

    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
                                          "process": "default",
                                          "bytes": 100,
                                          "threshold": 110,
                                          "objects": []})])

    updated = update(tests, log_0)
    assert not updated


def test_update_leak_total_3():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"""
leak-total: 100""")]

    log_0 = suite_log([("mozleak_total", {"scope": "path/to/",
                                          "process": "default",
                                          "bytes": 1000,
                                          "threshold": 100,
                                          "objects": []})])

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.get("leak-threshold") == ['default:51200']


def test_update_leak_total_4():
    tests = [("path/to/test.htm", [test_id], "testharness", b""),
             ("path/to/__dir__", [dir_id], None, b"""
leak-total: 110""")]

    log_0 = suite_log([
        ("lsan_leak", {"scope": "path/to/",
                       "frames": ["foo", "bar"]}),
        ("mozleak_total", {"scope": "path/to/",
                           "process": "default",
                           "bytes": 100,
                           "threshold": 110,
                           "objects": []})])

    updated = update(tests, log_0)
    new_manifest = updated[0][1]

    assert not new_manifest.is_empty
    assert new_manifest.modified
    assert new_manifest.has_key("leak-threshold") is False


class TestStep(Step):
    __test__ = False

    def create(self, state):
        tests = [("path/to/test.htm", [test_id], "testharness", "")]
        state.foo = create_test_manifest(tests)


class UpdateRunner(StepRunner):
    steps = [TestStep]


def test_update_pickle():
    logger = structuredlog.StructuredLogger("expected_test")
    wpt_root = os.path.abspath(os.path.join(here,
                                            os.pardir,
                                            os.pardir,
                                            os.pardir,
                                            os.pardir))
    args = {
        "test_paths": {
            "/": wptcommandline.TestRoot(wpt_root, wpt_root),
        },
        "abort": False,
        "continue": False,
        "sync": False,
    }
    args2 = args.copy()
    args2["abort"] = True
    wptupdate = WPTUpdate(logger, **args2)
    wptupdate = WPTUpdate(logger, runner_cls=UpdateRunner, **args)
    wptupdate.run()


def test_update_serialize_quoted():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  expected: "ERROR"
  [test1]
    expected:
     if os == "linux": ["PASS", "FAIL"]
     "ERROR"
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "OK"})],
                      run_info={"os": "linux"})
    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "OK"})],
                      run_info={"os": "linux"})
    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "ERROR"}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "OK"})],
                      run_info={"os": "win"})

    updated = update(tests, log_0, log_1, log_2, full_update=True, update_intermittent=True)


    manifest_str = wptmanifest.serialize(updated[0][1].node,
                                         skip_empty_data=True)
    assert manifest_str == """[test.htm]
  [test1]
    expected:
      if os == "linux": [PASS, FAIL]
      ERROR
"""


def test_update_serialize_unquoted():
    tests = [("path/to/test.htm", [test_id], "testharness",
              b"""[test.htm]
  expected: ERROR
  [test1]
    expected:
     if os == "linux": [PASS, FAIL]
     ERROR
""")]

    log_0 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "OK"})],
                      run_info={"os": "linux"})
    log_1 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "FAIL",
                                        "expected": "PASS",
                                        "known_intermittent": ["FAIL"]}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "OK"})],
                      run_info={"os": "linux"})
    log_2 = suite_log([("test_start", {"test": test_id}),
                       ("test_status", {"test": test_id,
                                        "subtest": "test1",
                                        "status": "ERROR"}),
                       ("test_end", {"test": test_id,
                                     "expected": "ERROR",
                                     "status": "OK"})],
                      run_info={"os": "win"})

    updated = update(tests, log_0, log_1, log_2, full_update=True, update_intermittent=True)


    manifest_str = wptmanifest.serialize(updated[0][1].node,
                                         skip_empty_data=True)
    assert manifest_str == """[test.htm]
  [test1]
    expected:
      if os == "linux": [PASS, FAIL]
      ERROR
"""
