SERVER-7695 Add ISO versions of date expressions.

Implements expressions for dealing with week as described in the
ISO-8601 standard (https://en.wikipedia.org/wiki/ISO_week_date).

Also implements the %G, %V, and %u specifiers for $dateToString.

Closes #1068

3 files changed, 478 insertions, 0 deletions

diff --git a/jstests/aggregation/bugs/server7695_isodates.js b/jstests/aggregation/bugs/server7695_isodates.js
new file mode 100644
index 00000000000..b6a97fe1ec0
--- /dev/null
+++ b/jstests/aggregation/bugs/server7695_isodates.js
@@ -0,0 +1,286 @@
+// SERVER-7695: Add $isoWeek, $isoWeekYear, and $isoDayOfWeek aggregation expressions.
+
+(function() {
+    "use strict";
+    var coll = db.server7695;
+    var testOpCount = 0;
+
+    coll.drop();
+
+    // Seed collection so that the pipeline will execute.
+    assert.writeOK(coll.insert({}));
+
+    // Helper for testing that op returns expResult.
+    function testOp(op, value, expResult) {
+        testOpCount++;
+        var pipeline = [{$project: {_id: 0, result: {}}}];
+        pipeline[0].$project.result[op] = value;
+        var msg = "Exptected {" + op + ": " + value + "} to equal: " + expResult;
+        var res = coll.runCommand('aggregate', {pipeline: pipeline});
+
+        // in the case of $dateToString the date is on property date
+        var date = value.date || value;
+        if (date.valueOf() < 0 && _isWindows() && res.code === 16422) {
+            // some versions of windows (but not all) fail with dates before 1970
+            print("skipping test of " + date.tojson() +
+                  " because system doesn't support old dates");
+            return;
+        }
+
+        if (date.valueOf() / 1000 < -2 * 1024 * 1024 * 1024 && res.code == 16421) {
+            // we correctly detected that we are outside of the range of a 32-bit time_t
+            print("skipping test of " + date.tojson() + " because it is outside of time_t range");
+            return;
+        }
+
+        assert.eq(res.result[0].result, expResult, pipeline);
+    }
+
+    // While development, there was a bug which caused an error with $dateToString if the order of
+    // %V and %G changed, so I added this test to prevent regression.
+    testOp('$dateToString', {date: new Date("1900-12-31T23:59:59Z"), format: "%V-%G"}, "01-1901");
+    // This was failing, but it shouldn't as it is the same as above, only rotated.
+    testOp('$dateToString', {date: new Date("1900-12-31T23:59:59Z"), format: "%G-%V"}, "1901-01");
+
+    // 1900 is special because it's devisible by 4 and by 100 but not 400 so it's not a leap year.
+    // 2000 is special, because it's devisible by 4, 100, 400 and so it is a leap year.
+    var years = {
+        common: [
+            1900,  // Starting and ending on Monday (special).
+            2002,  // Starting and ending on Tuesday.
+            2014,  // Starting and ending on Wednesday.
+            2015,  // Starting and ending on Thursday.
+            2010,  // Starting and ending on Friday.
+            2011,  // Starting and ending on Saturday.
+            2006,  // Starting and ending on Sunday.
+        ],
+        leap: [
+            1996,  // Starting on Monday, ending on Tuesday.
+            2008,  // Starting on Tuesday, ending on Wednesday.
+            1992,  // Starting on Wednesday, ending on Thursday.
+            2004,  // Starting on Thursday, ending on Friday.
+            2016,  // Starting on Friday, ending on Saturday.
+            2000,  // Starting on Saturday, ending on Sunday (special).
+            2012,  // Starting on Sunday, ending on Monday.
+        ],
+        commonAfterLeap: [
+            2001,  // Starting and ending on Monday.
+            2013,  // Starting and ending on Tuesday.
+            1997,  // Starting and ending on Wednesday.
+            2009,  // Starting and ending on Thursday.
+            1993,  // Starting and ending on Friday.
+            2005,  // Starting and ending on Saturday.
+            2017,  // Starting and ending on Sunday.
+        ],
+    };
+
+    var MONDAY = 1;
+    var TUESDAY = 2;
+    var WEDNESDAY = 3;
+    var THURSDAY = 4;
+    var FRIDAY = 5;
+    var SATURDAY = 6;
+    var SUNDAY = 7;
+
+    // Make numbers at least with two digits with 0 in front if shorter than two digits
+    function padded(val) {
+        return ("00" + val).slice(-2);
+    }
+    function getNewYear(year) {
+        return new Date("" + year + "-01-01T00:00:00Z");
+    }
+    function getEndOfFirstWeekInYear(year, day) {
+        return new Date("" + year + "-01-" + (padded(7 - day + 1)) + "T23:59:59Z");
+    }
+    function getStartOfSecondWeekInYear(year, day) {
+        return new Date("" + year + "-01-" + (padded(7 - day + 2)) + "T00:00:00Z");
+    }
+    function getBirthday(year) {
+        return new Date("" + year + "-07-05T21:36:00+02:00");
+    }
+    function getEndOfSecondToLastWeekInYear(year, day, type) {
+        if (type === 'leap') {
+            return new Date("" + year + "-12-" + padded(31 - day - 1) + "T23:59:59Z");
+        } else {
+            return new Date("" + year + "-12-" + padded(31 - day) + "T23:59:59Z");
+        }
+    }
+    function getStartOfLastWeekInYear(year, day, type) {
+        if (type === 'leap') {
+            return new Date("" + year + "-12-" + padded(31 - day) + "T00:00:00Z");
+        } else {
+            return new Date("" + year + "-12-" + padded(31 - day + 1) + "T00:00:00Z");
+        }
+    }
+    function getNewYearsEve(year) {
+        return new Date("" + year + "-12-31T23:59:59Z");
+    }
+    function shiftWeekday(dayOfWeek, daysToAdd) {
+        return ((dayOfWeek - 1 + daysToAdd) % 7) + 1;
+    }
+
+    ['common', 'leap', 'commonAfterLeap'].forEach(function(type) {
+        years[type].forEach(function(year, day) {
+            // forEach starts indexing at zero but weekdays start with Monday on 1 so we add +1.
+            var day = day + 1;
+            var newYear = getNewYear(year);
+            var endOfFirstWeekInYear = getEndOfFirstWeekInYear(year, day);
+            var startOfSecondWeekInYear = getStartOfSecondWeekInYear(year, day);
+            var birthday = getBirthday(year);
+            var endOfSecondToLastWeekInYear = getEndOfSecondToLastWeekInYear(year, day, type);
+            var startOfLastWeekInYear = getStartOfLastWeekInYear(year, day, type);
+            var newYearsEve = getNewYearsEve(year);
+
+            testOp('$isoDayOfWeek', newYear, day);
+            testOp('$isoDayOfWeek', endOfFirstWeekInYear, SUNDAY);
+            testOp('$isoDayOfWeek', startOfSecondWeekInYear, MONDAY);
+            testOp('$isoDayOfWeek', endOfSecondToLastWeekInYear, SUNDAY);
+            testOp('$isoDayOfWeek', startOfLastWeekInYear, MONDAY);
+            if (type === 'leap') {
+                testOp('$isoDayOfWeek', newYearsEve, shiftWeekday(day, 1));
+            } else {
+                testOp('$isoDayOfWeek', newYearsEve, day);
+            }
+
+            if (type === 'leap') {
+                testOp('$isoDayOfWeek', birthday, shiftWeekday(day, 4));
+            } else {
+                testOp('$isoDayOfWeek', birthday, shiftWeekday(day, 3));
+            }
+
+            testOp('$isoWeekYear', birthday, year);
+            // In leap years staring on Thursday, the birthday is in week 28, every year else it is
+            // in week 27.
+            if (type === 'leap' && day === THURSDAY) {
+                testOp('$isoWeek', birthday, 28);
+            } else {
+                testOp('$isoWeek', birthday, 27);
+            }
+
+            if (day <= THURSDAY) {
+                // A year starting between Monday and Thursday will always start in week 1.
+                testOp('$isoWeek', newYear, 1);
+                testOp('$isoWeekYear', newYear, year);
+                testOp('$isoWeek', endOfFirstWeekInYear, 1);
+                testOp('$isoWeekYear', endOfFirstWeekInYear, year);
+                testOp('$isoWeek', startOfSecondWeekInYear, 2);
+                testOp('$isoWeekYear', startOfSecondWeekInYear, year);
+                testOp('$dateToString',
+                       {format: '%G-W%V-%u', date: newYear},
+                       "" + year + "-W01-" + day);
+            } else if (day == FRIDAY || (day == SATURDAY && type === 'commonAfterLeap')) {
+                // A year starting on Friday will always start with week 53 of the previous year.
+                // A common year starting on a Saturday and after a leap year will also start with
+                // week 53 of the previous year.
+                testOp('$isoWeek', newYear, 53);
+                testOp('$isoWeekYear', newYear, year - 1);
+                testOp('$isoWeek', endOfFirstWeekInYear, 53);
+                testOp('$isoWeekYear', endOfFirstWeekInYear, year - 1);
+                testOp('$isoWeek', startOfSecondWeekInYear, 1);
+                testOp('$isoWeekYear', startOfSecondWeekInYear, year);
+                testOp('$dateToString',
+                       {format: '%G-W%V-%u', date: newYear},
+                       "" + (year - 1) + "-W53-" + day);
+            } else {
+                // A year starting on Saturday (except after a leap year) or Sunday will always
+                // start with week 52 of the previous year.
+                testOp('$isoWeek', newYear, 52);
+                testOp('$isoWeekYear', newYear, year - 1);
+                testOp('$isoWeek', endOfFirstWeekInYear, 52);
+                testOp('$isoWeekYear', endOfFirstWeekInYear, year - 1);
+                testOp('$isoWeek', startOfSecondWeekInYear, 1);
+                testOp('$isoWeekYear', startOfSecondWeekInYear, year);
+                testOp('$dateToString',
+                       {format: '%G-W%V-%u', date: newYear},
+                       "" + (year - 1) + "-W52-" + day);
+            }
+
+            if (type === 'leap') {
+                if (day <= TUESDAY) {
+                    // A leap year starting between Monday and Tuesday will always end in week 1 of
+                    // the next year.
+                    testOp('$isoWeek', newYearsEve, 1);
+                    testOp('$isoWeekYear', newYearsEve, year + 1);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 52);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 1);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year + 1);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year + 1) + "-W01-" + shiftWeekday(day, 1));
+                } else if (day <= THURSDAY) {
+                    // A leap year starting on Wednesday or Thursday will always end with week 53.
+                    testOp('$isoWeek', newYearsEve, 53);
+                    testOp('$isoWeekYear', newYearsEve, year);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 52);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 53);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year) + "-W53-" + shiftWeekday(day, 1));
+                } else if (day <= SATURDAY) {
+                    // A leap year starting on Friday or Sarturday will always and with week 52
+                    testOp('$isoWeek', newYearsEve, 52);
+                    testOp('$isoWeekYear', newYearsEve, year);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 51);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 52);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year) + "-W52-" + shiftWeekday(day, 1));
+                } else {
+                    // A leap year starting on Sunday will always end with week 1
+                    testOp('$isoWeek', newYearsEve, 1);
+                    testOp('$isoWeekYear', newYearsEve, year + 1);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 51);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 52);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year + 1) + "-W01-" + shiftWeekday(day, 1));
+                }
+            } else {
+                if (day <= WEDNESDAY) {
+                    // A common year starting between Monday and Wednesday will always end in week 1
+                    // of the next year.
+                    testOp('$isoWeek', newYearsEve, 1);
+                    testOp('$isoWeekYear', newYearsEve, year + 1);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 52);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 1);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year + 1);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year + 1) + "-W01-" + day);
+                } else if (day === THURSDAY) {
+                    // A common year starting on Thursday will always end with week 53.
+                    testOp('$isoWeek', newYearsEve, 53);
+                    testOp('$isoWeekYear', newYearsEve, year);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 52);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 53);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year) + "-W53-" + day);
+                } else {
+                    // A common year starting on between Friday and Sunday will always end with week
+                    // 52.
+                    testOp('$isoWeek', newYearsEve, 52);
+                    testOp('$isoWeekYear', newYearsEve, year);
+                    testOp('$isoWeek', endOfSecondToLastWeekInYear, 51);
+                    testOp('$isoWeekYear', endOfSecondToLastWeekInYear, year);
+                    testOp('$isoWeek', startOfLastWeekInYear, 52);
+                    testOp('$isoWeekYear', startOfLastWeekInYear, year);
+                    testOp('$dateToString',
+                           {format: '%G-W%V-%u', date: newYearsEve},
+                           "" + (year) + "-W52-" + day);
+                }
+            }
+        });
+    });
+    assert.eq(testOpCount, 485, 'Expected 485 tests to run');
+})();
diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp
index e94cd7b7467..7dc91bdd4fe 100644
--- a/src/mongo/db/pipeline/expression.cpp
+++ b/src/mongo/db/pipeline/expression.cpp
@@ -26,6 +26,7 @@
  * it in the license file.
  */
 
+
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/pipeline/expression.h"
@@ -997,6 +998,9 @@ void ExpressionDateToString::validateFormat(const std::string& format) {
             case 'j':
             case 'w':
             case 'U':
+            case 'G':
+            case 'V':
+            case 'u':
                 break;
             default:
                 uasserted(18536,
@@ -1060,6 +1064,15 @@ string ExpressionDateToString::formatDate(const string& format,
             case 'U':  // Week
                 insertPadded(formatted, ExpressionWeek::extract(tm), 2);
                 break;
+            case 'G':  // Iso year of week
+                insertPadded(formatted, ExpressionIsoWeekYear::extract(tm), 4);
+                break;
+            case 'V':  // Iso week
+                insertPadded(formatted, ExpressionIsoWeek::extract(tm), 2);
+                break;
+            case 'u':  // Iso day of week
+                insertPadded(formatted, ExpressionIsoDayOfWeek::extract(tm), 1);
+                break;
             default:
                 // Should never happen as format is pre-validated
                 invariant(false);
@@ -3284,6 +3297,158 @@ const char* ExpressionWeek::getOpName() const {
     return "$week";
 }
 
+/* ------------------------- ExpressionIsoDayOfWeek --------------------- */
+
+Value ExpressionIsoDayOfWeek::evaluateInternal(Variables* vars) const {
+    Value date(vpOperand[0]->evaluateInternal(vars));
+    return Value(extract(date.coerceToTm()));
+}
+
+int ExpressionIsoDayOfWeek::extract(const tm& tm) {
+    // translate from sunday=0 … saturday=6 to monday=1 … sunday=7
+    return (tm.tm_wday - 7) % 7 + 7;
+}
+
+REGISTER_EXPRESSION(isoDayOfWeek, ExpressionIsoDayOfWeek::parse);
+const char* ExpressionIsoDayOfWeek::getOpName() const {
+    return "$isoDayOfWeek";
+}
+
+/* ------------------------- ExpressionIsoWeekYear ---------------------- */
+
+Value ExpressionIsoWeekYear::evaluateInternal(Variables* vars) const {
+    Value date(vpOperand[0]->evaluateInternal(vars));
+    return Value(extract(date.coerceToTm()));
+}
+
+int ExpressionIsoWeekYear::extract(const tm& tm) {
+    if (tm.tm_mon > 0 && tm.tm_mon < 11) {
+        // If month is between February and November, it is just the year given.
+        return tm.tm_year + 1900;
+    } else if (tm.tm_mon == 0) {
+        // In January we need to check if the week belongs to previous year.
+        int isoWeek = ExpressionIsoWeek::extract(tm);
+        if (isoWeek > 51) {  // Weeks 52 and 53 belong to the previous year.
+            return tm.tm_year + 1900 - 1;
+        } else {  // All other weeks belong to given year.
+            return tm.tm_year + 1900;
+        }
+    } else {
+        // A week 1 in December belongs to the next year.
+        int isoWeek = ExpressionIsoWeek::extract(tm);
+        if (isoWeek == 1) {
+            return tm.tm_year + 1900 + 1;
+        } else {
+            return tm.tm_year + 1900;
+        }
+    }
+}
+
+REGISTER_EXPRESSION(isoWeekYear, ExpressionIsoWeekYear::parse);
+const char* ExpressionIsoWeekYear::getOpName() const {
+    return "$isoWeekYear";
+}
+
+/* ------------------------- ExpressionIsoWeek -------------------------- */
+
+namespace {
+bool isLeapYear(int year) {
+    if (year % 4 != 0) {
+        // Not a leap year because a leap year must be divisable by 4.
+        return false;
+    } else if (year % 100 != 0) {
+        // Every year that is divisable by 100 is a leap year.
+        return true;
+    } else if (year % 400 != 0) {
+        // Every 400th year is not a leap year, althoght it is divisable by 4.
+        return false;
+    } else {
+        // Every year divisable by 4 but not 400 is a leap year.
+        return true;
+    }
+}
+
+int lastWeek(int year) {
+    // Create YYYY-12-31T23:59:59 so only 1 second left to new year.
+    struct tm tm = {};
+    tm.tm_year = year - 1900;
+    tm.tm_mon = 11;
+    tm.tm_mday = 31;
+    tm.tm_hour = 23;
+    tm.tm_min = 59;
+    tm.tm_sec = 59;
+    mktime(&tm);
+
+    // From: https://en.wikipedia.org/wiki/ISO_week_date#Last_week :
+    // If 31 December is on a Monday, Tuesday or Wednesday, it is in week 01 of the next year. If
+    // it is on a Thursday, it is in week 53 of the year just ending; if on a Friday it is in week
+    // 52 (or 53 if the year just ending is a leap year); if on a Saturday or Sunday, it is in week
+    // 52 of the year just ending.
+    if (tm.tm_wday > 0 && tm.tm_wday < 4) {  // Mon(1), Tue(2), and Wed(3)
+        return 1;
+    } else if (tm.tm_wday == 4) {  // Thu (4)
+        return 53;
+    } else if (tm.tm_wday == 5) {  // Fri (5)
+        // On Fri it's week 52 for non leap years and 53 for leap years.
+        // https://en.wikipedia.org/wiki/Leap_year#Algorithm
+        if (isLeapYear(year)) {
+            return 53;
+        } else {
+            return 52;
+        }
+    } else {  // Sat (6) or Sun (0)
+        return 52;
+    }
+}
+}
+
+Value ExpressionIsoWeek::evaluateInternal(Variables* vars) const {
+    Value date(vpOperand[0]->evaluateInternal(vars));
+    return Value(extract(date.coerceToTm()));
+}
+
+// Quote https://en.wikipedia.org/wiki/ISO_week_date :
+// Weeks start with Monday. The first week of a year is the week that contains the first Thursday of
+// the year (and, hence, always contains 4 January).
+int ExpressionIsoWeek::extract(const tm& tm) {
+    // Calculation taken from:
+    // https://en.wikipedia.org/wiki/ISO_week_date#Calculating_the_week_number_of_a_given_date
+    //
+    // week(date) = floor( (ordinal(data) - weekday(date) + 10) / 7 )
+    //
+    // The first week must contain the first Thursday, therefore the `+ 10` after subtracting the
+    // weekday. Example: 2016-01-07 is the first Thursday
+    // ordinal(2016-01-07) => 7
+    // weekday(2016-01-07) => 4
+    //
+    // floor((7-4+10)/7) = floor(13/7) => 1
+    //
+    // week(date)    = isoWeek
+    // ordinal(date) = isoDayOfYear
+    // weekday(date) = isoDayOfWeek
+    int isoDayOfWeek = ExpressionIsoDayOfWeek::extract(tm);
+    int isoDayOfYear = tm.tm_yday + 1;
+    int isoWeek = (isoDayOfYear - isoDayOfWeek + 10) / 7;
+
+    // There is no week 0, so it must be the last week of the previous year.
+    if (isoWeek < 1) {
+        return lastWeek(tm.tm_year + 1900 - 1);
+        // If the calculated week is 53 and bigger than the last week, than it is the first week of
+        // the
+        // next year.
+    } else if (isoWeek == 53 && isoWeek > lastWeek(tm.tm_year + 1900)) {
+        return 1;
+        // It is just the week calculated
+    } else {
+        return isoWeek;
+    }
+}
+
+REGISTER_EXPRESSION(isoWeek, ExpressionIsoWeek::parse);
+const char* ExpressionIsoWeek::getOpName() const {
+    return "$isoWeek";
+}
+
 /* ------------------------- ExpressionYear ----------------------------- */
 
 Value ExpressionYear::evaluateInternal(Variables* vars) const {
diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h
index ae6adb7e7a6..fe3e57b0d4a 100644
--- a/src/mongo/db/pipeline/expression.h
+++ b/src/mongo/db/pipeline/expression.h
@@ -1303,6 +1303,33 @@ public:
 };
 
 
+class ExpressionIsoWeekYear final : public ExpressionFixedArity<ExpressionIsoWeekYear, 1> {
+public:
+    Value evaluateInternal(Variables* vars) const final;
+    const char* getOpName() const final;
+
+    static int extract(const tm& tm);
+};
+
+
+class ExpressionIsoDayOfWeek final : public ExpressionFixedArity<ExpressionIsoDayOfWeek, 1> {
+public:
+    Value evaluateInternal(Variables* vars) const final;
+    const char* getOpName() const final;
+
+    static int extract(const tm& tm);
+};
+
+
+class ExpressionIsoWeek final : public ExpressionFixedArity<ExpressionIsoWeek, 1> {
+public:
+    Value evaluateInternal(Variables* vars) const final;
+    const char* getOpName() const final;
+
+    static int extract(const tm& tm);
+};
+
+
 class ExpressionYear final : public ExpressionFixedArity<ExpressionYear, 1> {
 public:
     Value evaluateInternal(Variables* vars) const final;