/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*
 * This file tests our indexing prowess.  This includes both our ability to
 *  properly be triggered by events taking place in thunderbird as well as our
 *  ability to correctly extract/index the right data.
 * In general, if these tests pass, things are probably working quite well.
 *
 * This test has local, IMAP online, IMAP offline, and IMAP online-become-offline
 *  variants.  See the text_index_messages_*.js files.
 *
 * Things we don't test that you think we might test:
 * - Full-text search.  Happens in query testing.
 */

load("resources/glodaTestHelper.js");

ChromeUtils.import("resource:///modules/MailUtils.js");
ChromeUtils.import("resource://gre/modules/NetUtil.jsm");
ChromeUtils.import("resource://gre/modules/Services.jsm");

// Whether we can expect fulltext results
var expectFulltextResults = true;

/**
 * Should we force our folders offline after we have indexed them once.  We do
 * this in the online_to_offline test variant.
 */
var goOffline = false;

/* ===== Indexing Basics ===== */

/**
 * Index a message, wait for a commit, make sure the header gets the property
 *  set correctly.  Then modify the message, verify the dirty property shows
 *  up, flush again, and make sure the dirty property goes clean again.
 */
function* test_pending_commit_tracker_flushes_correctly() {
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer(msgSet, {augment: true});

  // before the flush, there should be no gloda-id property
  let msgHdr = msgSet.getMsgHdr(0);
  // get it as a string to make sure it's empty rather than possessing a value
  Assert.equal(msgHdr.getStringProperty("gloda-id"), "");

  yield wait_for_gloda_db_flush();

  // after the flush there should be a gloda-id property and it should
  //  equal the gloda id
  let gmsg = msgSet.glodaMessages[0];
  Assert.equal(msgHdr.getUint32Property("gloda-id"), gmsg.id);

  // make sure no dirty property was written...
  Assert.equal(msgHdr.getStringProperty("gloda-dirty"), "");

  // modify the message
  msgSet.setRead(true);
  yield wait_for_gloda_indexer(msgSet);

  // now there should be a dirty property and it should be 1...
  Assert.equal(msgHdr.getUint32Property("gloda-dirty"),
               GlodaMsgIndexer.kMessageDirty);

  // flush
  yield wait_for_gloda_db_flush();

  // now dirty should be 0 and the gloda id should still be the same
  Assert.equal(msgHdr.getUint32Property("gloda-dirty"),
               GlodaMsgIndexer.kMessageClean);
  Assert.equal(msgHdr.getUint32Property("gloda-id"), gmsg.id);
}

/**
 * Make sure that PendingCommitTracker causes a msgdb commit to occur so that
 *  if the nsIMsgFolder's msgDatabase attribute has already been nulled
 *  (which is normally how we force a msgdb commit), that the changes to the
 *  header actually hit the disk.
 */
function* test_pending_commit_causes_msgdb_commit() {
  // new message, index it
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer(msgSet, {augment: true});

  // force the msgDatabase closed; the sqlite commit will not yet have occurred
  get_real_injection_folder(folder).msgDatabase = null;
  // make the commit happen, this causes the header to get set.
  yield wait_for_gloda_db_flush();
  // Force a GC.  this will kill off the header and the database, losing data
  //  if we are not protecting it.
  Components.utils.forceGC();

  // now retrieve the header and make sure it has the gloda id set!
  let msgHdr = msgSet.getMsgHdr(0);
  Assert.equal(msgHdr.getUint32Property("gloda-id"), msgSet.glodaMessages[0].id);
}

/**
 * Give the indexing sweep a workout.
 *
 * This includes:
 * - Basic indexing sweep across never-before-indexed folders.
 * - Indexing sweep across folders with just some changes.
 * - Filthy pass.
 */
function* test_indexing_sweep() {
  // -- Never-before-indexed folders
  mark_sub_test_start("never before indexed folders");
  // turn off event-driven indexing
  configure_gloda_indexing({event: false});

  let [folderA, setA1, setA2] = make_folder_with_sets([{count: 3},
                                                       {count: 2}]);
  yield wait_for_message_injection();
  let [folderB, setB1, setB2] = make_folder_with_sets([{count: 3},
                                                       {count: 2}]);
  yield wait_for_message_injection();
  let [folderC, setC1, setC2] = make_folder_with_sets([{count: 3},
                                                       {count: 2}]);
  yield wait_for_message_injection();

  // Make sure that event-driven job gets nuked out of existence
  GlodaIndexer.purgeJobsUsingFilter(() => true);

  // turn on event-driven indexing again; this will trigger a sweep.
  configure_gloda_indexing({event: true});
  GlodaMsgIndexer.indexingSweepNeeded = true;
  yield wait_for_gloda_indexer([setA1, setA2, setB1, setB2, setC1, setC2]);


  // -- Folders with some changes, pending commits
  mark_sub_test_start("folders with some changes, pending commits");
  // indexing off
  configure_gloda_indexing({event: false});

  setA1.setRead(true);
  setB2.setRead(true);

  // indexing on, killing all outstanding jobs, trigger sweep
  GlodaIndexer.purgeJobsUsingFilter(() => true);
  configure_gloda_indexing({event: true});
  GlodaMsgIndexer.indexingSweepNeeded = true;

  yield wait_for_gloda_indexer([setA1, setB2]);


  // -- Folders with some changes, no pending commits
  mark_sub_test_start("folders with some changes, no pending commits");
  // force a commit to clear out our pending commits
  yield wait_for_gloda_db_flush();
  // indexing off
  configure_gloda_indexing({event: false});

  setA2.setRead(true);
  setB1.setRead(true);

  // indexing on, killing all outstanding jobs, trigger sweep
  GlodaIndexer.purgeJobsUsingFilter(() => true);
  configure_gloda_indexing({event: true});
  GlodaMsgIndexer.indexingSweepNeeded = true;

  yield wait_for_gloda_indexer([setA2, setB1]);


  // -- Filthy foldering indexing
  // Just mark the folder filthy and make sure that we reindex everyone.
  // IMPORTANT!  The trick of marking the folder filthy only works because
  //  we flushed/committed the database above; the PendingCommitTracker
  //  is not aware of bogus filthy-marking of folders.
  // We leave the verification of the implementation details to
  //  test_index_sweep_folder.js.
  mark_sub_test_start("filthy folder indexing");
  let glodaFolderC = Gloda.getFolderForFolder(
                       get_real_injection_folder(folderC));
  glodaFolderC._dirtyStatus = glodaFolderC.kFolderFilthy;
  mark_action("actual", "marked gloda folder dirty", [glodaFolderC]);
  GlodaMsgIndexer.indexingSweepNeeded = true;
  yield wait_for_gloda_indexer([setC1, setC2]);

  // -- Forced folder indexing.
  var callbackInvoked = false;
  mark_sub_test_start("forced folder indexing");
  GlodaMsgIndexer.indexFolder(get_real_injection_folder(folderA), {
    force: true,
    callback: function() {
      callbackInvoked = true;
    }});
  yield wait_for_gloda_indexer([setA1, setA2]);
  Assert.ok(callbackInvoked);
}


/**
 * We used to screw up and downgrade filthy folders to dirty if we saw an event
 *  happen in the folder before we got to the folder; this tests that we no
 *  longer do that.
 */
function* test_event_driven_indexing_does_not_mess_with_filthy_folders() {
  // add a folder with a message.
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer([msgSet]);

  // fake marking the folder filthy.
  let glodaFolder = Gloda.getFolderForFolder(get_real_injection_folder(folder));
  glodaFolder._dirtyStatus = glodaFolder.kFolderFilthy;

  // generate an event in the folder
  msgSet.setRead(true);
  // make sure the indexer did not do anything and the folder is still filthy.
  yield wait_for_gloda_indexer([]);
  Assert.equal(glodaFolder._dirtyStatus, glodaFolder.kFolderFilthy);
  // also, the message should not have actually gotten marked dirty
  Assert.equal(msgSet.getMsgHdr(0).getUint32Property("gloda-dirty"), 0);

  // let's make the message un-read again for consistency with the gloda state
  msgSet.setRead(false);
  // make the folder dirty and let an indexing sweep take care of this so we
  //  don't get extra events in subsequent tests.
  glodaFolder._dirtyStatus = glodaFolder.kFolderDirty;
  GlodaMsgIndexer.indexingSweepNeeded = true;
  // (the message won't get indexed though)
  yield wait_for_gloda_indexer([]);
}

function* test_indexing_never_priority() {

  // add a folder with a bunch of messages
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();

  // index it, and augment the msgSet with the glodaMessages array
  // for later use by sqlExpectCount
  yield wait_for_gloda_indexer([msgSet], {augment: true});

  // explicitly tell gloda to never index this folder
  let XPCOMFolder = get_real_injection_folder(folder);
  let glodaFolder = Gloda.getFolderForFolder(XPCOMFolder);
  GlodaMsgIndexer.setFolderIndexingPriority(XPCOMFolder,
                                            glodaFolder.kIndexingNeverPriority);

  // verify that the setter and getter do the right thing
  Assert.equal(glodaFolder.indexingPriority, glodaFolder.kIndexingNeverPriority);

  // check that existing message is marked as deleted
  yield wait_for_gloda_indexer([], {deleted: [msgSet]});

  // make sure the deletion hit the database
  yield sqlExpectCount(1,
    "SELECT COUNT(*) from folderLocations WHERE id = ? AND indexingPriority = ?",
     glodaFolder.id, glodaFolder.kIndexingNeverPriority);

  // add another message
  make_new_sets_in_folder(folder, [{count: 1}]);
  yield wait_for_message_injection();

  // make sure that indexing returns nothing
  GlodaMsgIndexer.indexingSweepNeeded = true;
  yield wait_for_gloda_indexer([]);
}

function* test_setting_indexing_priority_never_while_indexing() {

  if (!message_injection_is_local())
    return;

  // Configure the gloda indexer to hang while streaming the message.
  configure_gloda_indexing({hangWhile: "streaming"});

  // create a folder with a message inside.
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();

  yield wait_for_indexing_hang();

  // explicitly tell gloda to never index this folder
  let XPCOMFolder = get_real_injection_folder(folder);
  let glodaFolder = Gloda.getFolderForFolder(XPCOMFolder);
  GlodaMsgIndexer.setFolderIndexingPriority(XPCOMFolder,
                                            glodaFolder.kIndexingNeverPriority);

  // reset indexing to not hang
  configure_gloda_indexing({});

  // sorta get the event chain going again...
  resume_from_simulated_hang(true);

  // Because the folder was dirty it should actually end up getting indexed,
  //  so in the end the message will get indexed.  Also, make sure a cleanup
  //  was observed.
  yield wait_for_gloda_indexer([], {cleanedUp: 1});
}

/* ===== Threading / Conversation Grouping ===== */

var gSynMessages = [];
function allMessageInSameConversation(aSynthMessage, aGlodaMessage, aConvID) {
  if (aConvID === undefined)
    return aGlodaMessage.conversationID;
  Assert.equal(aConvID, aGlodaMessage.conversationID);
  // Cheat and stash the synthetic message (we need them for one of the IMAP
  // tests)
  gSynMessages.push(aSynthMessage);
  return aConvID;
}

/**
 * Test our conversation/threading logic in the straight-forward direct
 *  reply case, the missing intermediary case, and the siblings with missing
 *  parent case.  We also test all permutations of receipt of those messages.
 * (Also tests that we index new messages.)
 */
function* test_threading() {
  mark_sub_test_start("direct reply");
  yield indexAndPermuteMessages(scenarios.directReply,
                                allMessageInSameConversation);
  mark_sub_test_start("missing intermediary");
  yield indexAndPermuteMessages(scenarios.missingIntermediary,
                                allMessageInSameConversation);
  mark_sub_test_start("siblings missing parent");
  yield indexAndPermuteMessages(scenarios.siblingsMissingParent,
                                allMessageInSameConversation);
}

/**
 * Test the bit that says "if we're fulltext-indexing the message and we
 *  discover it didn't have any attachments, clear the attachment bit from the
 *  message header".
 */
function* test_attachment_flag() {
  // create a synthetic message with an attachment that won't normally be listed
  // in the attachment pane (Content-Disposition: inline, no filename, and
  // displayable inline)
  let smsg = msgGen.makeMessage({
    name: 'test message with part 1.2 attachment',
    attachments: [{ body: 'attachment',
                    filename: '',
                    format: '' }],
  });
  // save it off for test_attributes_fundamental_from_disk
  let msgSet = new SyntheticMessageSet([smsg]);
  let folder = fundamentalFolderHandle = make_empty_folder();
  yield add_sets_to_folders(folder, [msgSet]);

  // if we need to go offline, let the indexing pass run, then force us offline
  if (goOffline) {
    yield wait_for_gloda_indexer(msgSet);
    yield make_folder_and_contents_offline(folder);
    // now the next indexer wait will wait for the next indexing pass...
  }

  yield wait_for_gloda_indexer(msgSet,
                               {verifier: verify_attachment_flag});

}

function verify_attachment_flag(smsg, gmsg) {
  // -- attachments. We won't have these if we don't have fulltext results
  if (expectFulltextResults) {
    Assert.equal(gmsg.attachmentNames.length, 0);
    Assert.equal(gmsg.attachmentInfos.length, 0);
    Assert.equal(false, gmsg.folderMessage.flags & Ci.nsMsgMessageFlags.Attachment);
  }
}
/* ===== Fundamental Attributes (per fundattr.js) ===== */

/**
 * Save the synthetic message created in test_attributes_fundamental for the
 *  benefit of test_attributes_fundamental_from_disk.
 */
var fundamentalSyntheticMessage;
var fundamentalFolderHandle;
/**
 * We're saving this one so that we can move the message later and verify that
 * the attributes are consistent.
 */
var fundamentalMsgSet;
var fundamentalGlodaMsgAttachmentUrls;
/**
 * Save the resulting gloda message id corresponding to the
 *  fundamentalSyntheticMessage so we can use it to query the message from disk.
 */
var fundamentalGlodaMessageId;

/**
 * Test that we extract the 'fundamental attributes' of a message properly
 *  'Fundamental' in this case is talking about the attributes defined/extracted
 *  by gloda's fundattr.js and perhaps the core message indexing logic itself
 *  (which show up as kSpecial* attributes in fundattr.js anyways.)
 */
function* test_attributes_fundamental() {
  // create a synthetic message with attachment
  let smsg = msgGen.makeMessage({
    name: 'test message',
    bodyPart: new SyntheticPartMultiMixed([
      new SyntheticPartLeaf({body: 'I like cheese!'}),
      msgGen.makeMessage({ body: { body: 'I like wine!' }}), // that's one attachment
    ]),
    attachments: [
      {filename: 'bob.txt', body: 'I like bread!'} // and that's another one
    ],
  });
  // save it off for test_attributes_fundamental_from_disk
  fundamentalSyntheticMessage = smsg;
  let msgSet = new SyntheticMessageSet([smsg]);
  fundamentalMsgSet = msgSet;
  let folder = fundamentalFolderHandle = make_empty_folder();
  yield add_sets_to_folders(folder, [msgSet]);

  // if we need to go offline, let the indexing pass run, then force us offline
  if (goOffline) {
    yield wait_for_gloda_indexer(msgSet);
    yield make_folder_and_contents_offline(folder);
    // now the next indexer wait will wait for the next indexing pass...
  }

  yield wait_for_gloda_indexer(msgSet, { verifier: verify_attributes_fundamental });

}

function verify_attributes_fundamental(smsg, gmsg) {
  // save off the message id for test_attributes_fundamental_from_disk
  fundamentalGlodaMessageId = gmsg.id;
  if (gmsg.attachmentInfos) {
    fundamentalGlodaMsgAttachmentUrls =
      gmsg.attachmentInfos.map(att => att.url);
  } else {
    fundamentalGlodaMsgAttachmentUrls = [];
  }

  Assert.equal(gmsg.folderURI,
               get_real_injection_folder(fundamentalFolderHandle).URI);

  // -- subject
  Assert.equal(smsg.subject, gmsg.conversation.subject);
  Assert.equal(smsg.subject, gmsg.subject);

  // -- contact/identity information
  // - from
  // check the e-mail address
  Assert.equal(gmsg.from.kind, "email");
  Assert.equal(smsg.fromAddress, gmsg.from.value);
  // check the name
  Assert.equal(smsg.fromName, gmsg.from.contact.name);

  // - to
  Assert.equal(smsg.toAddress, gmsg.to[0].value);
  Assert.equal(smsg.toName, gmsg.to[0].contact.name);

  // date
  Assert.equal(smsg.date.valueOf(), gmsg.date.valueOf());

  // -- message ID
  Assert.equal(smsg.messageId, gmsg.headerMessageID);

  // -- attachments. We won't have these if we don't have fulltext results
  if (expectFulltextResults) {
    Assert.equal(gmsg.attachmentTypes.length, 1);
    Assert.equal(gmsg.attachmentTypes[0], "text/plain");
    Assert.equal(gmsg.attachmentNames.length, 1);
    Assert.equal(gmsg.attachmentNames[0], "bob.txt");

    let expectedInfos = [
      // the name for that one is generated randomly
      { contentType: "message/rfc822" },
      { name: "bob.txt", contentType: "text/plain" },
    ];
    let expectedSize = 14;
    Assert.equal(gmsg.attachmentInfos.length, 2);
    for (let [i, attInfos] of gmsg.attachmentInfos.entries()) {
      for (let k in expectedInfos[i])
        Assert.equal(attInfos[k], expectedInfos[i][k]);
      // because it's unreliable and depends on the platform
      Assert.ok(Math.abs(attInfos.size - expectedSize) <= 2);
      // check that the attachment URLs are correct
      let channel = NetUtil.newChannel({
         uri: attInfos.url,
         loadingPrincipal: Services.scriptSecurityManager.getSystemPrincipal(),
         securityFlags: Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
         contentPolicyType: Ci.nsIContentPolicy.TYPE_OTHER
      });

      try {
        // will throw if the URL is invalid
        channel.open();
      } catch (e) {
        do_throw(new Error("Invalid attachment URL"));
      }
    }
  }
  else {
    // Make sure we don't actually get attachments!
    Assert.equal(gmsg.attachmentTypes, null);
    Assert.equal(gmsg.attachmentNames, null);
  }
}

/**
 * We now move the message into another folder, wait for it to be indexed,
 * and make sure the magic url getter for GlodaAttachment returns a proper
 * URL.
 */
function* test_moved_message_attributes() {
  if (!expectFulltextResults)
    return;

  // Don't ask me why, let destFolder = make_empty_folder would result in a
  // random error when running test_index_messages_imap_offline.js ...
  let [destFolder, ignoreSet] = make_folder_with_sets([{count: 2}]);
  fundamentalFolderHandle = destFolder;
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer([ignoreSet]);

  // this is a fast move (third parameter set to true)
  yield async_move_messages(fundamentalMsgSet, destFolder, true);

  yield wait_for_gloda_indexer(fundamentalMsgSet, {
    verifier: function (newSynMsg, newGlodaMsg) {
      // verify we still have the same number of attachments
      Assert.equal(fundamentalGlodaMsgAttachmentUrls.length,
        newGlodaMsg.attachmentInfos.length);
      for (let [i, attInfos] of newGlodaMsg.attachmentInfos.entries()) {
        // verify the url has changed
        Assert.notEqual(fundamentalGlodaMsgAttachmentUrls[i], attInfos.url);
        // and verify that the new url is still valid
        let channel = NetUtil.newChannel({
           uri: attInfos.url,
           loadingPrincipal: Services.scriptSecurityManager.getSystemPrincipal(),
           securityFlags: Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
           contentPolicyType: Ci.nsIContentPolicy.TYPE_OTHER
        });
        try {
          channel.open();
        } catch (e) {
          do_throw(new Error("Invalid attachment URL"));
        }
      }
    },
    fullyIndexed: 0,
  });
}

/**
 * We want to make sure that all of the fundamental properties also are there
 *  when we load them from disk.  Nuke our cache, query the message back up.
 *  We previously used getMessagesByMessageID to get the message back, but he
 *  does not perform a full load-out like a query does, so we need to use our
 *  query mechanism for this.
 */
function test_attributes_fundamental_from_disk() {
  nukeGlodaCachesAndCollections();

  let query = Gloda.newQuery(Gloda.NOUN_MESSAGE).id(fundamentalGlodaMessageId);
  queryExpect(query, [fundamentalSyntheticMessage],
              verify_attributes_fundamental_from_disk,
              function (smsg) { return smsg.messageId; } );
  return false;
}

/**
 * We are just a wrapper around verify_attributes_fundamental, adapting the
 *  return callback from getMessagesByMessageID.
 *
 * @param aGlodaMessageLists This should be [[theGlodaMessage]].
 */
function verify_attributes_fundamental_from_disk(aGlodaMessage) {
  // return the message id for test_attributes_fundamental_from_disk's benefit
  verify_attributes_fundamental(fundamentalSyntheticMessage,
                                aGlodaMessage);
  return aGlodaMessage.headerMessageID;
}

/* ===== Explicit Attributes (per explattr.js) ===== */

/**
 * Test the attributes defined by explattr.js.
 */
function* test_attributes_explicit() {
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer(msgSet, {augment: true});
  let gmsg = msgSet.glodaMessages[0];

  // -- Star
  mark_sub_test_start("Star");
  msgSet.setStarred(true);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.starred, true);

  msgSet.setStarred(false);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.starred, false);

  // -- Read / Unread
  mark_sub_test_start("Read/Unread");
  msgSet.setRead(true);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.read, true);

  msgSet.setRead(false);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.read, false);

  // -- Tags
  mark_sub_test_start("Tags");
  // note that the tag service does not guarantee stable nsIMsgTag references,
  //  nor does noun_tag go too far out of its way to provide stability.
  //  However, it is stable as long as we don't spook it by bringing new tags
  //  into the equation.
  let tagOne = TagNoun.getTag("$label1");
  let tagTwo = TagNoun.getTag("$label2");

  msgSet.addTag(tagOne.key);
  yield wait_for_gloda_indexer(msgSet);
  Assert.notEqual(gmsg.tags.indexOf(tagOne), -1);

  msgSet.addTag(tagTwo.key);
  yield wait_for_gloda_indexer(msgSet);
  Assert.notEqual(gmsg.tags.indexOf(tagOne), -1);
  Assert.notEqual(gmsg.tags.indexOf(tagTwo), -1);

  msgSet.removeTag(tagOne.key);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.tags.indexOf(tagOne), -1);
  Assert.notEqual(gmsg.tags.indexOf(tagTwo), -1);

  msgSet.removeTag(tagTwo.key);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.tags.indexOf(tagOne), -1);
  Assert.equal(gmsg.tags.indexOf(tagTwo), -1);

  // -- Replied To

  // -- Forwarded
}


/**
 * Test non-query-able attributes
 */
function* test_attributes_cant_query() {
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer(msgSet, {augment: true});
  let gmsg = msgSet.glodaMessages[0];

  // -- Star
  mark_sub_test_start("Star");
  msgSet.setStarred(true);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.starred, true);

  msgSet.setStarred(false);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.starred, false);

  // -- Read / Unread
  mark_sub_test_start("Read/Unread");
  msgSet.setRead(true);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.read, true);

  msgSet.setRead(false);
  yield wait_for_gloda_indexer(msgSet);
  Assert.equal(gmsg.read, false);

  let readDbAttr = Gloda.getAttrDef(Gloda.BUILT_IN, "read");
  let readId = readDbAttr.id;

  yield sqlExpectCount(0, "SELECT COUNT(*) FROM messageAttributes WHERE attributeID = ?1",
                       readId);

  // -- Replied To

  // -- Forwarded
}

/**
 * Have the participants be in our addressbook prior to indexing so that we can
 *  verify that the hand-off to the addressbook indexer does not cause breakage.
 */
function* test_people_in_addressbook() {
  var senderPair = msgGen.makeNameAndAddress(),
      recipPair = msgGen.makeNameAndAddress();

  // - add both people to the address book
  makeABCardForAddressPair(senderPair);
  makeABCardForAddressPair(recipPair);

  let [folder, msgSet] = make_folder_with_sets([
    { count: 1, to: [recipPair], from: senderPair }]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer(msgSet, {augment: true});
  let gmsg = msgSet.glodaMessages[0],
      senderIdentity = gmsg.from,
      recipIdentity = gmsg.to[0];

  Assert.notEqual(senderIdentity.contact, null);
  Assert.ok(senderIdentity.inAddressBook);

  Assert.notEqual(recipIdentity.contact, null);
  Assert.ok(recipIdentity.inAddressBook);
}

/* ===== fulltexts Indexing ===== */

/**
 * Make sure that we are using the saneBodySize flag.  This is basically the
 *  test_sane_bodies test from test_mime_emitter but we pull the indexedBodyText
 *  off the message to check and also make sure that the text contents slice
 *  off the end rather than the beginning.
 */
function* test_streamed_bodies_are_size_capped() {
  if (!expectFulltextResults)
    return;

  let hugeString =
    "qqqqxxxx qqqqxxx qqqqxxx qqqqxxx qqqqxxx qqqqxxx qqqqxxx \r\n";
  const powahsOfTwo = 10;
  for (let i = 0; i < powahsOfTwo; i++) {
    hugeString = hugeString + hugeString;
  }
  let bodyString = "aabb" + hugeString + "xxyy";

  let synMsg = gMessageGenerator.makeMessage(
    {body: {body: bodyString, contentType: "text/plain"}});
  let msgSet = new SyntheticMessageSet([synMsg]);
  let folder = make_empty_folder();
  yield add_sets_to_folder(folder, [msgSet]);

  if (goOffline) {
    yield wait_for_gloda_indexer(msgSet);
    yield make_folder_and_contents_offline(folder);
  }

  yield wait_for_gloda_indexer(msgSet, {augment: true});
  let gmsg = msgSet.glodaMessages[0];
  Assert.ok(gmsg.indexedBodyText.startsWith("aabb"));
  Assert.ok(!gmsg.indexedBodyText.includes("xxyy"));

  if (gmsg.indexedBodyText.length > (20 * 1024 + 58 + 10))
    do_throw("indexed body text is too big! (" + gmsg.indexedBodyText.length +
             ")");
}


/* ===== Message Deletion ===== */
/**
 * Test actually deleting a message on a per-message basis (not just nuking the
 *  folder like emptying the trash does.)
 *
 * Logic situations:
 * - Non-last message in a conversation, twin.
 * - Non-last message in a conversation, not a twin.
 * - Last message in a conversation
 */
function* test_message_deletion() {
  mark_sub_test_start("non-last message in conv, twin");
  // create and index two messages in a conversation
  let [folder, convSet] = make_folder_with_sets([{count: 2, msgsPerThread: 2}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer([convSet], {augment: true});

  // Twin the first message in a different folder owing to our reliance on
  //  message-id's in the SyntheticMessageSet logic.  (This is also why we broke
  //  up the indexing waits too.)
  let twinFolder = make_empty_folder();
  let twinSet = new SyntheticMessageSet([convSet.synMessages[0]]);
  yield add_sets_to_folder(twinFolder, [twinSet]);
  yield wait_for_gloda_indexer([twinSet], {augment: true});

  // Split the conv set into two helper sets...
  let firstSet = convSet.slice(0, 1); // the twinned first message in the thread
  let secondSet = convSet.slice(1, 2); // the un-twinned second thread message

  // make sure we can find the message (paranoia)
  let firstQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE);
  firstQuery.id(firstSet.glodaMessages[0].id);
  let firstColl = queryExpect(firstQuery, firstSet);
  yield false; // queryExpect is async but returns a value...

  // delete it (not trash! delete!)
  yield async_delete_messages(firstSet);
  // which should result in an apparent deletion
  yield wait_for_gloda_indexer([], {deleted: firstSet});
  // and our collection from that query should now be empty
  Assert.equal(firstColl.items.length, 0);

  // make sure it no longer shows up in a standard query
  firstColl = queryExpect(firstQuery, []);
  yield false; // queryExpect is async

  // make sure it shows up in a privileged query
  let privQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
                      noDbQueryValidityConstraints: true,
                    });
  let firstGlodaId = firstSet.glodaMessages[0].id;
  privQuery.id(firstGlodaId);
  queryExpect(privQuery, firstSet);
  yield false; // queryExpect is async

  // force a deletion pass
  GlodaMsgIndexer.indexingSweepNeeded = true;
  yield wait_for_gloda_indexer([]);

  // Make sure it no longer shows up in a privileged query; since it has a twin
  //  we don't need to leave it as a ghost.
  queryExpect(privQuery, []);
  yield false; // queryExpect is async

  // make sure the messagesText entry got blown away
  yield sqlExpectCount(0, "SELECT COUNT(*) FROM messagesText WHERE docid = ?1",
                       firstGlodaId);

  // make sure the conversation still exists...
  let conv = twinSet.glodaMessages[0].conversation;
  let convQuery = Gloda.newQuery(Gloda.NOUN_CONVERSATION);
  convQuery.id(conv.id);
  let convColl = queryExpect(convQuery, [conv]);
  yield false; // queryExpect is async


  // -- non-last message, no longer a twin => ghost
  mark_sub_test_start("non-last message in conv, no longer a twin");

  // make sure nuking the twin didn't somehow kill them both
  let twinQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE);
  // (let's search on the message-id now that there is no ambiguity.)
  twinQuery.headerMessageID(twinSet.synMessages[0].messageId);
  let twinColl = queryExpect(twinQuery, twinSet);
  yield false; // queryExpect is async

  // delete the twin
  yield async_delete_messages(twinSet);
  // which should result in an apparent deletion
  yield wait_for_gloda_indexer([], {deleted: twinSet});
  // it should disappear from the collection
  Assert.equal(twinColl.items.length, 0);

  // no longer show up in the standard query
  twinColl = queryExpect(twinQuery, []);
  yield false; // queryExpect is async

  // still show up in a privileged query
  privQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
                               noDbQueryValidityConstraints: true,
                             });
  privQuery.headerMessageID(twinSet.synMessages[0].messageId);
  queryExpect(privQuery, twinSet);
  yield false; // queryExpect is async

  // force a deletion pass
  GlodaMsgIndexer.indexingSweepNeeded = true;
  yield wait_for_gloda_indexer([]);

  // The message should be marked as a ghost now that the deletion pass.
  // Ghosts have no fulltext rows, so check for that.
  yield sqlExpectCount(0, "SELECT COUNT(*) FROM messagesText WHERE docid = ?1",
                       twinSet.glodaMessages[0].id);

  // it still should show up in the privileged query; it's a ghost!
  let privColl = queryExpect(privQuery, twinSet);
  yield false; // queryExpect is async
  // make sure it looks like a ghost.
  let twinGhost = privColl.items[0];
  Assert.equal(twinGhost._folderID, null);
  Assert.equal(twinGhost._messageKey, null);

  // make sure the conversation still exists...
  queryExpect(convQuery, [conv]);
  yield false; // queryExpect is async


  // -- non-last message, not a twin
  // This should blow away the message, the ghosts, and the conversation.
  mark_sub_test_start("last message in conv");

  // second message should still be around
  let secondQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE);
  secondQuery.headerMessageID(secondSet.synMessages[0].messageId);
  let secondColl = queryExpect(secondQuery, secondSet);
  yield false; // queryExpect is async

  // delete it and make sure it gets marked deleted appropriately
  yield async_delete_messages(secondSet);
  yield wait_for_gloda_indexer([], {deleted: secondSet});
  Assert.equal(secondColl.items.length, 0);

  // still show up in a privileged query
  privQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
                               noDbQueryValidityConstraints: true,
                             });
  privQuery.headerMessageID(secondSet.synMessages[0].messageId);
  queryExpect(privQuery, secondSet);
  yield false; // queryExpect is async

  // force a deletion pass
  GlodaMsgIndexer.indexingSweepNeeded = true;
  yield wait_for_gloda_indexer([]);

  // it should no longer show up in a privileged query; we killed the ghosts
  queryExpect(privQuery, []);
  yield false; // queryExpect is async

  // - the conversation should have disappeared too
  // (we have no listener to watch for it to have disappeared from convQuery but
  //  this is basically how glodaTestHelper does its thing anyways.)
  Assert.equal(convColl.items.length, 0);

  // make sure the query fails to find it too
  queryExpect(convQuery, []);
  yield false; // queryExpect is async


  // -- identity culling verification
  mark_sub_test_start("identity culling verification");
  // The identities associated with that message should no longer exist, nor
  //  should their contacts.

}

function* test_moving_to_trash_marks_deletion() {
  // create and index two messages in a conversation
  let [folder, msgSet] = make_folder_with_sets([{count: 2, msgsPerThread: 2}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer([msgSet], {augment: true});

  let convId = msgSet.glodaMessages[0].conversation.id;
  let firstGlodaId = msgSet.glodaMessages[0].id;
  let secondGlodaId = msgSet.glodaMessages[1].id;

  // move them to the trash.
  yield async_trash_messages(msgSet);

  // we do not index the trash folder so this should actually make them appear
  //  deleted to an unprivileged query.
  let msgQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE);
  msgQuery.id(firstGlodaId, secondGlodaId);
  queryExpect(msgQuery, []);
  yield false; // queryExpect is async

  // they will appear deleted after the events
  yield wait_for_gloda_indexer([], {deleted: msgSet});

  // force a sweep
  GlodaMsgIndexer.indexingSweepNeeded = true;
  // there should be no apparent change as the result of this pass
  // (well, the conversation will die, but we can't see that.)
  yield wait_for_gloda_indexer([]);

  // the conversation should be gone
  let convQuery = Gloda.newQuery(Gloda.NOUN_CONVERSATION);
  convQuery.id(convId);
  queryExpect(convQuery, []);
  yield false; // queryExpect is async

  // the messages should be entirely gone
  let msgPrivQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
                                      noDbQueryValidityConstraints: true,
                                    });
  msgPrivQuery.id(firstGlodaId, secondGlodaId);
  queryExpect(msgPrivQuery, []);
  yield false; // queryExpect is async
}

/**
 * Deletion that occurs because a folder got deleted.
 *  There is no hand-holding involving the headers that were in the folder.
 */
function* test_folder_nuking_message_deletion() {
  // create and index two messages in a conversation
  let [folder, msgSet] = make_folder_with_sets([{count: 2, msgsPerThread: 2}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer([msgSet], {augment: true});

  let convId = msgSet.glodaMessages[0].conversation.id;
  let firstGlodaId = msgSet.glodaMessages[0].id;
  let secondGlodaId = msgSet.glodaMessages[1].id;

  // Delete the folder
  yield async_delete_folder(folder);
  // That does generate the deletion events if the messages were in-memory,
  //  which these are.
  yield wait_for_gloda_indexer([], {deleted: msgSet});

  // this should have caused us to mark all the messages as deleted; the
  //  messages should no longer show up in an unprivileged query
  let msgQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE);
  msgQuery.id(firstGlodaId, secondGlodaId);
  queryExpect(msgQuery, []);
  yield false; // queryExpect is async

  // force a sweep
  GlodaMsgIndexer.indexingSweepNeeded = true;
  // there should be no apparent change as the result of this pass
  // (well, the conversation will die, but we can't see that.)
  yield wait_for_gloda_indexer([]);

  // the conversation should be gone
  let convQuery = Gloda.newQuery(Gloda.NOUN_CONVERSATION);
  convQuery.id(convId);
  queryExpect(convQuery, []);
  yield false; // queryExpect is async

  // the messages should be entirely gone
  let msgPrivQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE, {
                                      noDbQueryValidityConstraints: true,
                                    });
  msgPrivQuery.id(firstGlodaId, secondGlodaId);
  queryExpect(msgPrivQuery, []);
  yield false; // queryExpect is async
}

/* ===== Folder Move/Rename/Copy (Single and Nested) ===== */

function get_nsIMsgFolder(aFolder) {
  if (!(aFolder instanceof Ci.nsIMsgFolder))
    return MailUtils.getFolderForURI(aFolder);
  else
    return aFolder;
}

function get_testFolder(aFolder) {
  if ((typeof aFolder) != "string")
    return aFolder.URI;
  else
    return aFolder;
}

function* test_folder_deletion_nested() {
  // add a folder with a bunch of messages
  let [folder1, msgSet1] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();

  let [folder2, msgSet2] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();

  // index these folders, and augment the msgSet with the glodaMessages array
  // for later use by sqlExpectCount
  yield wait_for_gloda_indexer([msgSet1, msgSet2], { augment: true });
  // the move has to be performed after the indexing, because otherwise, on
  // IMAP, the moved message header are different entities and it's not msgSet2
  // that ends up indexed, but the fresh headers
  yield move_folder(folder2, folder1);

  // add a trash folder, and move folder1 into it
  let trash = make_empty_folder(null, [Ci.nsMsgFolderFlags.Trash]);
  yield move_folder(folder1, trash);

  let descendentFolders = get_nsIMsgFolder(trash).descendants;
  let folders = Array.from(fixIterator(descendentFolders, Ci.nsIMsgFolder));
  Assert.equal(folders.length, 2);
  let [newFolder1, newFolder2] = folders;

  let glodaFolder1 = Gloda.getFolderForFolder(newFolder1);
  let glodaFolder2 = Gloda.getFolderForFolder(newFolder2);

  // verify that Gloda properly marked this folder as not to be indexed anymore
  Assert.equal(glodaFolder1.indexingPriority, glodaFolder1.kIndexingNeverPriority);

  // check that existing message is marked as deleted
  yield wait_for_gloda_indexer([], {deleted: [msgSet1, msgSet2]});

  // make sure the deletion hit the database
  yield sqlExpectCount(1,
    "SELECT COUNT(*) from folderLocations WHERE id = ? AND indexingPriority = ?",
     glodaFolder1.id, glodaFolder1.kIndexingNeverPriority);
  yield sqlExpectCount(1,
    "SELECT COUNT(*) from folderLocations WHERE id = ? AND indexingPriority = ?",
     glodaFolder2.id, glodaFolder2.kIndexingNeverPriority);

  if (_messageInjectionSetup.injectionConfig.mode == "local") {
    // add another message
    make_new_sets_in_folder(newFolder1, [{count: 1}]);
    yield wait_for_message_injection();
    make_new_sets_in_folder(newFolder2, [{count: 1}]);
    yield wait_for_message_injection();

    // make sure that indexing returns nothing
    GlodaMsgIndexer.indexingSweepNeeded = true;
    yield wait_for_gloda_indexer([]);
  }
}

/* ===== IMAP Nuances ===== */

/**
 * Verify that for IMAP folders we still see an index a message that is added
 *  as read.
 */
function* test_imap_add_unread_to_folder() {
  if (message_injection_is_local())
    return;

  let [srcFolder, msgSet] = make_folder_with_sets([{count: 1, read: true}]);
  yield wait_for_message_injection();
  yield wait_for_gloda_indexer(msgSet);
}

/* ===== Message Moving ===== */

/**
 * Moving a message between folders should result in us knowing that the message
 *  is in the target location.
 */
function* test_message_moving() {
  // - inject and insert
  // source folder with the message we care about
  let [srcFolder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();
  // dest folder with some messages in it to test some wacky local folder moving
  //  logic.  (Local moves try and update the correspondence immediately.)
  let [destFolder, ignoreSet] = make_folder_with_sets([{count: 2}]);
  yield wait_for_message_injection();

  // (we want the gloda message mapping...)
  yield wait_for_gloda_indexer([msgSet, ignoreSet], {augment: true});
  let gmsg = msgSet.glodaMessages[0];
  // save off the message key so we can make sure it changes.
  let oldMessageKey = msgSet.getMsgHdr(0).messageKey;

  // - fastpath (offline) move it to a new folder
  mark_sub_test_start("initial move");
  yield async_move_messages(msgSet, destFolder, true);

  // - make sure gloda sees it in the new folder
  // Since we are doing offline IMAP moves, the fast-path should be taken and
  //  so we should receive an itemsModified notification without a call to
  //  Gloda.grokNounItem.
  yield wait_for_gloda_indexer(msgSet, {fullyIndexed: 0});

  Assert.equal(gmsg.folderURI,
               get_real_injection_folder(destFolder).URI);

  // - make sure the message key is correct!
  Assert.equal(gmsg.messageKey, msgSet.getMsgHdr(0).messageKey);
  // (sanity check that the messageKey actually changed for the message...)
  Assert.notEqual(gmsg.messageKey, oldMessageKey);

  // - make sure the indexer's _keyChangedBatchInfo dict is empty
  for (let evilKey in GlodaMsgIndexer._keyChangedBatchInfo) {
    let evilValue = GlodaMsgIndexer._keyChangedBatchInfo[evilKey];
    mark_failure(["GlodaMsgIndexer._keyChangedBatchInfo should be empty but",
                  "has key:",  evilKey, "and value:", evilValue, "."]);
  }

  // - slowpath (IMAP online) move it back to its origin folder
  mark_sub_test_start("move it back");
  yield async_move_messages(msgSet, srcFolder, false);
  // In the IMAP case we will end up reindexing the message because we will
  //  not be able to fast-path, but the local case will still be fast-pathed.
  yield wait_for_gloda_indexer(msgSet,
                               {fullyIndexed: message_injection_is_local() ?
                                                0 : 1});
  Assert.equal(gmsg.folderURI,
               get_real_injection_folder(srcFolder).URI);
  Assert.equal(gmsg.messageKey, msgSet.getMsgHdr(0).messageKey);
}

/**
 * Moving a gloda-indexed message out of a filthy folder should result in the
 *  destination message not having a gloda-id.
 */

/* ===== Message Copying ===== */


/* ===== Sweep Complications ==== */

/**
 * Make sure that a message indexed by event-driven indexing does not
 *  get reindexed by sweep indexing that follows.
 */
function* test_sweep_indexing_does_not_reindex_event_indexed() {
  let [folder, msgSet] = make_folder_with_sets([{count: 1}]);
  yield wait_for_message_injection();

  // wait for the event sweep to complete
  yield wait_for_gloda_indexer([msgSet]);

  // force a sweep of the folder
  GlodaMsgIndexer.indexFolder(get_real_injection_folder(folder));
  yield wait_for_gloda_indexer([]);
}

/**
 * Verify that moving apparently gloda-indexed messages from a filthy folder or
 *  one that simply should not be gloda indexed does not result in the target
 *  messages having the gloda-id property on them.  To avoid messing with too
 *  many invariants we do the 'folder should not be gloda indexed' case.
 * Uh, and of course, the message should still get indexed once we clear the
 *  filthy gloda-id off of it given that it is moving from a folder that is not
 *  indexed to one that is indexed.
 */
function* test_filthy_moves_slash_move_from_unindexed_to_indexed() {
  // - inject
  // the source folder needs a flag so we don't index it
  let srcFolder = make_empty_folder(null, [Ci.nsMsgFolderFlags.Junk]);
  // the destination folder has to be something we want to index though;
  let destFolder = make_empty_folder();
  let [msgSet] = make_new_sets_in_folder(srcFolder, [{count: 1}]);
  yield wait_for_message_injection();

  // - mark with a bogus gloda-id
  msgSet.getMsgHdr(0).setUint32Property("gloda-id", 9999);

  // - disable event driven indexing so we don't get interference from indexing
  configure_gloda_indexing({event: false});

  // - move
  yield async_move_messages(msgSet, destFolder);

  // - verify the target has no gloda-id!
  mark_action("actual", "checking", [msgSet.getMsgHdr(0)]);
  Assert.equal(msgSet.getMsgHdr(0).getUint32Property("gloda-id"), 0);

  // - re-enable indexing and let the indexer run
  // (we don't want to affect other tests)
  configure_gloda_indexing({});
  yield wait_for_gloda_indexer([msgSet]);
}

var tests = [
  test_pending_commit_tracker_flushes_correctly,
  test_pending_commit_causes_msgdb_commit,
  test_indexing_sweep,
  test_event_driven_indexing_does_not_mess_with_filthy_folders,

  test_threading,
  test_attachment_flag,
  test_attributes_fundamental,
  test_moved_message_attributes,
  test_attributes_fundamental_from_disk,
  test_attributes_explicit,
  test_attributes_cant_query,

  test_people_in_addressbook,

  test_streamed_bodies_are_size_capped,

  test_imap_add_unread_to_folder,
  test_message_moving,

  test_message_deletion,
  test_moving_to_trash_marks_deletion,
  test_folder_nuking_message_deletion,

  test_sweep_indexing_does_not_reindex_event_indexed,

  test_filthy_moves_slash_move_from_unindexed_to_indexed,

  test_indexing_never_priority,
  test_setting_indexing_priority_never_while_indexing,

  test_folder_deletion_nested,
];
