entry_dedupe: link, published, and published minute heuristics. #371

Author: lemon24

src/reader/plugins/entry_dedupe.py

@@ -156,6 +156,7 @@
 from functools import lru_cache
 from itertools import chain
 from itertools import islice
+from urllib.parse import urlparse
 
 from reader._storage._html_utils import strip_html
 from reader._utils import BetterStrPartial as partial
@@ -324,25 +325,56 @@ def group_entries(all_entries, new_entries, is_duplicate):
 
 
 def title_grouper(entries, new_entries):
+    return group_by(lambda e: tokenize_title(e.title), entries, new_entries)
+
+
+def link_grouper(entries, new_entries):
+    return group_by(lambda e: normalize_url(e.link), entries, new_entries)
+
+
+def normalize_url(url):
+    try:
+        url = urlparse(url)
+    except ValueError:
+        return None
+
+    scheme = url.scheme.lower()
+    if scheme == 'http':
+        scheme = 'https'
+
+    netloc = url.netloc.lower()
+    path = url.path.rstrip('/')
+
+    return url._replace(scheme=scheme, netloc=netloc, path=path).geturl()
+
+
+def published_grouper(entries, new_entries):
     def key(e):
-        return tokenize_title(e.title)
+        dt = e.published or e.updated
+        if not dt:
+            return None
+        return dt.isoformat(timespec='minutes')
 
     return group_by(key, entries, new_entries)
 
 
-GROUPERS = [title_grouper]
+def published_day_grouper(entries, new_entries):
+    def key(e):
+        dt = e.published or e.updated
+        if not dt:
+            return None
+        return dt.date()
+
+    return group_by(key, entries, new_entries)
+
+
+GROUPERS = [title_grouper, link_grouper, published_grouper, published_day_grouper]
 
 
 # entry (content) similarity
 
 
 def is_duplicate_entry(one, two):
-    # TODO: remove title checks once thresholds are increased for #371
-    if not one.title or not two.title:
-        return False
-    if tokenize_title(one.title) != tokenize_title(two.title):
-        return False
-
     one_fields = _content_fields(one)
     two_fields = _content_fields(two)
 

tests/test_plugins_entry_dedupe.py

@@ -29,40 +29,50 @@ def with_plugin():
     """Tell reader to use the plugin from the beginning."""
 
 
-def test_duplicates_are_deleted(reader, with_plugin, parser):
-    # detailed matching in test_is_duplicate_entry
+def test_only_duplicates_are_deleted(reader, parser):
+    # detailed/fuzzy content matching tested in test_is_duplicate*
 
     reader.add_feed(parser.feed(1))
 
-    yesterday = datetime(2010, 1, 1)
-    parser.entry(1, 1, yesterday, title='title', summary='value')
-    reader.update_feeds()
+    parser.entry(
+        1,
+        'different',
+        datetime(2010, 1, 1, 2, 3, 4),
+        title='title',
+        link='link',
+        summary='another',
+    )
+    parser.entry(1, 'title', title='title', summary='value')
+    parser.entry(1, 'title-x', summary='value')
+    parser.entry(1, 'link', link='link', summary='value')
+    parser.entry(1, 'link-x', link='link')
+    parser.entry(1, 'published', published=datetime(2010, 1, 1, 2, 3), summary='value')
+    parser.entry(1, 'published-x', published=datetime(2010, 1, 1, 2, 3))
+    parser.entry(1, 'published-day', datetime(2010, 1, 1), summary='value')
+    parser.entry(1, 'published-day-x', datetime(2010, 1, 1))
 
-    today = datetime(2010, 1, 2)
-    parser.entry(1, 2, today, title='title', summary='value')
     reader.update_feeds()
 
-    assert {e.id for e in reader.get_entries()} == {'1, 2'}
-
-
-def test_non_duplicates_are_ignored(reader, with_plugin, parser):
-    # detailed matching in test_is_duplicate_entry
-
-    reader.add_feed(parser.feed(1))
-
-    yesterday = datetime(2010, 1, 1)
-    parser.entry(1, 1, None, title=None)
-    parser.entry(1, 2, yesterday, title='title')
-    parser.entry(1, 3, yesterday, summary='value')
-    parser.entry(1, 4, yesterday, title='title', summary='another')
-    parser.entry(1, 5, yesterday, title='another', summary='value')
-    reader.update_feeds()
+    init_reader(reader)
 
-    today = datetime(2010, 1, 2)
-    parser.entry(1, 6, today, title='title', summary='value')
+    parser.entry(
+        1,
+        'entry',
+        datetime(2010, 1, 1, 2, 3, 4),
+        title='title',
+        link='link',
+        summary='value',
+    )
     reader.update_feeds()
 
-    assert {eval(e.id)[1] for e in reader.get_entries()} == {1, 2, 3, 4, 5, 6}
+    assert {e.id for e in reader.get_entries()} == {
+        'different',
+        'title-x',
+        'link-x',
+        'published-x',
+        'published-day-x',
+        'entry',
+    }
 
 
 def test_duplicates_in_another_feed_are_ignored(reader, with_plugin, parser):
@@ -270,11 +280,11 @@ def make_entry(title=None, summary=None, content=None):
 IS_DUPLICATE_ENTRY_DATA = [
     (make_entry(), make_entry(), False),
     (make_entry(title='title'), make_entry(title='title'), False),
-    (make_entry(summary='summary'), make_entry(summary='summary'), False),
+    (make_entry(summary='summary'), make_entry(summary='summary'), True),
     (
         make_entry(content=('value', 'text/html')),
         make_entry(content=('value', 'text/html')),
-        False,
+        True,
     ),
     (
         make_entry(title='title', summary='summary'),
@@ -284,7 +294,7 @@ def make_entry(title=None, summary=None, content=None):
     (
         make_entry(title='title', summary='summary'),
         make_entry(title='other', summary='summary'),
-        False,
+        True,
     ),
     (
         make_entry(title='title', summary='summary'),
@@ -299,7 +309,7 @@ def make_entry(title=None, summary=None, content=None):
     (
         make_entry(title='title', content=('value', 'text/html')),
         make_entry(title='other', content=('value', 'text/html')),
-        False,
+        True,
     ),
     (
         make_entry(title='title', content=('value', 'text/html')),