javascript - 从数组中删除精确的重复行

Question

我在Javascript中有以下列表。

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

此列表包含许多重复行。我想从中删除所有重复的行。

对象也可以有10个属性，因此检查每个属性不是解决方案。无论我上网的方式是一维数组还是删除特定属性的重复项。 如何维护包含所有唯一行的列表？

Answer 1

这将从您的对象数组中删除完全重复项。

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" },
     { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" },
     { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" },
     {NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required"},
     {NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate"}, 
     { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" },
     { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     {ID: "RID005",NAME: "CALCULATE FAST GROUP",EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" },
     {NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required"}, 
     {NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate"}];
     
var filtered = [...new Set(list.map(v => JSON.stringify(v)))];

console.log(filtered.map(v => JSON.parse(v)));

Answer 2

来自the MDN article on JSON.stringify：

  

不保证非数组对象的属性按任何特定顺序进行字符串化。不要依赖于字符串化内同一对象内的属性排序。

不应直接比较每个对象的JSON字符串，而应首先使用Object.entries获取[key, value]对数组，并按字符串值key对其进行排序。无论JSON.stringify实现如何，都可以安全地对这个排序的条目数组进行字符串化和比较。

请注意，我的示例使用了最新的ES6功能，包括Set和Object.entries。要将此代码转换为ES5并运行它，请访问https://babeljs.io/repl/。

function deepUnique (array) {
  return [...new Set(array.map(e => JSON.stringify(Object.entries(e).sort(compareEntriesByKey))))].map(e => objectFromEntries(JSON.parse(e)))
}

function compareEntriesByKey (a, b) {
  return a[0] < b[0] ? -1 : a[0] === b[0] ? 0 : 1
}

function objectFromEntries (entries) {
  let result = {}
  entries.forEach(e => result[e[0]] = e[1])
  return result
}

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

console.log(deepUnique(list))

在我发布我之前，感谢Kind user创建an answer using both JSON and Set。

Answer 3

对所有元素进行字符串化，同时检查是否存在类似元素，然后解析它们。不过，JSON.stringify()和JSON.parse()的速度并不快。如果您的对象非常大，请记住这一点。

var list = [{
    ID: "2566",
    NAME: "ENTERPRISE EXPENSE",
    EXECUTE: "LOAD_DIMENTION",
    TYPE: "PROCEDURE"
  },
  {
    ID: "1234",
    NAME: "LOAD EXPENSE FACT",
    EXECUTE: "LOAD_EXPENSE_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI0031",
    NAME: "LOAD HEAD COUNT",
    EXECUTE: "LOAD_HEADCOUNT_FACT",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RI04",
    NAME: "CALCULATE FAST",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "FORMULAS",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    ID: "RID005",
    NAME: "CALCULATE FAST GROUP",
    EXECUTE: "FF_CALC",
    TYPE: "STORED PROCEDURE"
  },
  {
    NAME: "Not Required",
    EXECUTE: "Not Required",
    TYPE: "Not Required"
  },
  {
    NAME: "Duplicate",
    EXECUTE: "Duplicate",
    TYPE: "Duplicate"
  }
];

function removeDuplicates(list) {
  return list.reduce(function(memo, obj) {
    var string = JSON.stringify(obj);

    if (memo.indexOf(string) === -1) {
      memo.push(string);
    }

    return memo;
  }, []).map(function(stringified) {
    return JSON.parse(stringified);
  });
}

list = removeDuplicates(list);
console.log(list);

Answer 4

以下代码段将通过使用JSON字符串化键和相应的对象项作为值构建哈希映射来删除重复项。通过不对值进行字符串化，我们保留其引用。因此，此处不使用JSON.parse()操作。

function removeDupes(a){
  var hash = a.reduce(function(h,o){
                        var os = JSON.stringify(o);
                        return h[os] ? h : (h[os] = o, h);
  },{});
  return Object.keys(hash)
               .map(k => hash[k]);
}

var list = [{     ID: "2566",
                NAME: "ENTERPRISE EXPENSE",
             EXECUTE: "LOAD_DIMENTION",
                TYPE: "PROCEDURE"
            },
            {     ID: "1234",
                NAME: "LOAD EXPENSE FACT",
             EXECUTE: "LOAD_EXPENSE_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RID005",
                NAME: "CALCULATE FAST GROUP",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            }
           ],
     res = removeDupes(list);
console.log(res);

但是，如果您想要删除所有重复项目，那么您仍然可以以类似的方式完成工作;

function removeDupes(a){
  var hash = a.reduce(function(h,o){
                        var os = JSON.stringify(o);
                        return h[os] ? (h[os] = false,h) : (h[os] = o, h);
                      },{});
  return Object.keys(hash)
               .reduce((r,k) => hash[k] === false ? r : r.concat(hash[k]),[]);
}

var list = [{     ID: "2566",
                NAME: "ENTERPRISE EXPENSE",
             EXECUTE: "LOAD_DIMENTION",
                TYPE: "PROCEDURE"
            },
            {     ID: "1234",
                NAME: "LOAD EXPENSE FACT",
             EXECUTE: "LOAD_EXPENSE_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI0031",
                NAME: "LOAD HEAD COUNT",
             EXECUTE: "LOAD_HEADCOUNT_FACT",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RI04",
                NAME: "CALCULATE FAST",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "FORMULAS",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                  ID: "RID005",
                NAME: "CALCULATE FAST GROUP",
             EXECUTE: "FF_CALC",
                TYPE: "STORED PROCEDURE"
            },
            {
                NAME: "Not Required",
             EXECUTE: "Not Required",
                TYPE: "Not Required"
            },
            {
                NAME: "Duplicate",
             EXECUTE: "Duplicate",
                TYPE: "Duplicate"
            }
           ],
     res = removeDupes(list);
console.log(res);

Answer 5

您可以使用只有排序键作为哈希的哈希表和具有相同键的对象的数组。

{
    "EXECUTE|ID|NAME|TYPE": [
        { ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" },
        { ID: "1234", /* ... */ },
        { ID: "RI0031", /* ... */ },
        { ID: "RI04", /* ... */ },
        { ID: "RID005", /* ... */ }
    ],
    "EXECUTE|NAME|TYPE": [
        { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" },
        { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" },
        { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }
    ]
}

对于过滤，您可以首先检查哈希表中的密钥，使用管道连接，然后如果找到，则迭代哈希表的数组并检查所有属性。如果不相等，则将实际对象推送到哈希表，然后返回true。

＆＃13;

＆＃13;

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" }, { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RID005", NAME: "CALCULATE FAST GROUP", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }],
    result = list.filter(function (hash) {
        return function (o) {
            var keys = Object.keys(o).sort(),
                key = keys.join('|');

            if (!hash[key]) {
                hash[key] = [o];
                return true;
            }
            if (!hash[key].some(function (p) { return keys.every(function (k) { return o[k] === p[k]; }); })) {
                hash[key].push(o);
                return true;
            }
        };
    }(Object.create(null)));

console.log(result);

＆＃13;

.as-console-wrapper { max-height: 100% !important; top: 0; }

＆＃13;

＆＃13;

＆＃13;

ES6与Map

＆＃13;

＆＃13;

var list = [{ ID: "2566", NAME: "ENTERPRISE EXPENSE", EXECUTE: "LOAD_DIMENTION", TYPE: "PROCEDURE" }, { ID: "1234", NAME: "LOAD EXPENSE FACT", EXECUTE: "LOAD_EXPENSE_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RI0031", NAME: "LOAD HEAD COUNT", EXECUTE: "LOAD_HEADCOUNT_FACT", TYPE: "STORED PROCEDURE" }, { ID: "RI04", NAME: "CALCULATE FAST", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "FORMULAS", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { ID: "RID005", NAME: "CALCULATE FAST GROUP", EXECUTE: "FF_CALC", TYPE: "STORED PROCEDURE" }, { NAME: "Not Required", EXECUTE: "Not Required", TYPE: "Not Required" }, { NAME: "Duplicate", EXECUTE: "Duplicate", TYPE: "Duplicate" }],
    result = list.filter((map => o => {
        var keys = Object.keys(o).sort(),
            key = keys.join('|');

        if (!map.has(key)) {
            map.set(key, [o]);
            return true;
        }
        if (!map.get(key).some(p => keys.every(k => o[k] === p[k]))) {
            map.get(key).push(o);
            return true;
        }
    })(new Map));

console.log(result);

＆＃13;

.as-console-wrapper { max-height: 100% !important; top: 0; }

＆＃13;

＆＃13;

＆＃13;