Skip to content

Commit ce8bf44

Browse files
amCap1712amCap1712
committed
Add indexing tests for entities using actual data
1 parent 55c5560 commit ce8bf44

19 files changed

Lines changed: 1212 additions & 8 deletions

‎sir/indexing.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ def _index_entity_process_wrapper(args, live=False):
199199
raise
200200

201201

202-
def index_entity(entity_name, bounds, data_queue):
202+
def index_entity(entity_name, bounds, data_queue, session=None):
203203
"""
204204
Retrieve rows for a single entity type identified by ``entity_name``,
205205
convert them to a dict with :func:`sir.indexing.query_result_to_dict` and
@@ -217,7 +217,7 @@ def index_entity(entity_name, bounds, data_queue):
217217
condition = and_(model.id >= lower_bound, model.id < upper_bound)
218218
else:
219219
condition = model.id >= lower_bound
220-
_query_database(entity_name, condition, data_queue)
220+
_query_database(entity_name, condition, data_queue, session)
221221

222222

223223
def live_index_entity(entity_name, ids, data_queue):

‎sir/util.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,20 +36,27 @@ def __str__(self):
3636
self.actual)
3737

3838

39-
def db_session():
39+
def engine():
4040
"""
41-
Creates a new :class:`sqla:sqlalchemy.orm.session.sessionmaker`.
41+
Create a new :class:`sqla:sqlalchemy.engine.Engine`.
4242
43-
:rtype: :class:`sqla:sqlalchemy.orm.session.sessionmaker`
43+
:rtype: :class:`sqla:sqlalchemy.engine.Engine`
4444
"""
4545
cget = partial(config.CFG.get, "database")
4646
cdict = {"username": cget("user")}
4747
for key in ["password", "host", "port"]:
4848
cdict[key] = cget(key)
4949
cdict["database"] = cget("dbname")
50-
e = create_engine(URL("postgresql", **cdict), server_side_cursors=False)
51-
S = sessionmaker(bind=e)
52-
return S
50+
return create_engine(URL("postgresql", **cdict), server_side_cursors=False)
51+
52+
53+
def db_session():
54+
"""
55+
Creates a new :class:`sqla:sqlalchemy.orm.session.sessionmaker`.
56+
57+
:rtype: :class:`sqla:sqlalchemy.orm.session.sessionmaker`
58+
"""
59+
return sessionmaker(bind=engine())
5360

5461

5562
@contextmanager

‎test/sql/annotation.sql‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
INSERT INTO editor (id, name, password, privs, email, website, bio, member_since,
2+
email_confirm_date, last_login_date, ha1)
3+
VALUES (1, 'new_editor', '{CLEARTEXT}password', 1+8+32+512, 'test@email.com', 'http://test.website',
4+
'biography', '1989-07-23', '2005-10-20', '2013-04-05', 'aa550c5b01407ef1f3f0d16daf9ec3c8');
5+
6+
INSERT INTO area (id, gid, name, type) VALUES
7+
(221, '8a754a16-0027-3a29-b6d7-2b40ea0481ed', 'United Kingdom', 1),
8+
(222, '489ce91b-6658-3307-9877-795b68554c98', 'United States', 1);
9+
INSERT INTO country_area (area) VALUES (221), (222);
10+
INSERT INTO iso_3166_1 (area, code) VALUES (221, 'GB'), (222, 'US');
11+
12+
INSERT INTO artist
13+
(id, gid, name, sort_name, type, gender, area,
14+
begin_area, end_area,
15+
begin_date_year, begin_date_month, begin_date_day,
16+
end_date_year, end_date_month, end_date_day, comment,
17+
last_updated)
18+
VALUES
19+
(3, '745c079d-374e-4436-9448-da92dedef3ce', 'Test Artist', 'Artist, Test', 1, 1, 221, 221, 221,
20+
2008, 01, 02, 2009, 03, 04, 'Yet Another Test Artist',
21+
'2009-07-09');
22+
23+
INSERT INTO artist (id, gid, name, sort_name) VALUES
24+
(4, '945c079d-374e-4436-9448-da92dedef3cf', 'Minimal Artist', 'Minimal Artist'),
25+
(5, 'dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958', 'Annotated Artist A', 'Annotated Artist A'),
26+
(6, 'ca4c2228-227c-4904-932a-dff442c091ea', 'Annotated Artist B', 'Annotated Artist B');
27+
28+
UPDATE artist_meta SET rating=70, rating_count=4 WHERE id=3;
29+
30+
INSERT INTO annotation (id, editor, text) VALUES (1, 1, 'Test annotation 1');
31+
INSERT INTO annotation (id, editor, text) VALUES (2, 1, 'Test annotation 2');
32+
INSERT INTO annotation (id, editor, text) VALUES (3, 1, 'Duplicate annotation');
33+
INSERT INTO annotation (id, editor, text) VALUES (4, 1, 'Duplicate annotation');
34+
35+
INSERT INTO artist_annotation (artist, annotation) VALUES (3, 1), (4, 2), (5, 3), (6, 4);
36+
37+
INSERT INTO artist_gid_redirect VALUES ('a4ef1d08-962e-4dd6-ae14-e42a6a97fc11', 3);

‎test/sql/area.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
INSERT INTO area (id, gid, name, type) VALUES
2+
( 13, '106e0bec-b638-3b37-b731-f53d507dc00e', 'Australia', 1),
3+
( 81, '85752fda-13c4-31a3-bee5-0e5cb1f51dad', 'Germany', 1),
4+
(107, '2db42837-c832-3c27-b4a3-08198f75693c', 'Japan', 1),
5+
(221, '8a754a16-0027-3a29-b6d7-2b40ea0481ed', 'United Kingdom', 1),
6+
(222, '489ce91b-6658-3307-9877-795b68554c98', 'United States', 1),
7+
(241, '89a675c2-3e37-3518-b83c-418bad59a85a', 'Europe', 1),
8+
(5126, '3f179da4-83c6-4a28-a627-e46b4a8ff1ed', 'Sydney', 3);
9+
INSERT INTO country_area (area) VALUES ( 13), ( 81), (107), (221), (222), (241);
10+
INSERT INTO iso_3166_1 (area, code) VALUES ( 13, 'AU'), ( 81, 'DE'), (107, 'JP'), (221, 'GB'), (222, 'US'), (241, 'XE');
11+
12+
INSERT INTO area_alias (id, name, sort_name, area, edits_pending)
13+
VALUES (1, 'オーストラリア', 'オーストラリア', 13, 0);
14+
15+
INSERT INTO link VALUES (118734, 356, NULL, NULL, NULL, NULL, NULL, NULL, 0, '2013-05-17 20:05:50.534145+00', FALSE);
16+
INSERT INTO l_area_area VALUES (4892, 118734, 13, 5126, 0, '2013-05-24 20:32:44.702487+00', 0, '', '');

‎test/sql/artist.sql‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
INSERT INTO editor (id, name, password, privs, email, website, bio, member_since,
2+
email_confirm_date, last_login_date, ha1)
3+
VALUES (1, 'new_editor', '{CLEARTEXT}password', 1+8+32+512, 'test@email.com', 'http://test.website',
4+
'biography', '1989-07-23', '2005-10-20', '2013-04-05', 'aa550c5b01407ef1f3f0d16daf9ec3c8');
5+
6+
INSERT INTO area (id, gid, name, type) VALUES
7+
(221, '8a754a16-0027-3a29-b6d7-2b40ea0481ed', 'United Kingdom', 1),
8+
(222, '489ce91b-6658-3307-9877-795b68554c98', 'United States', 1);
9+
INSERT INTO country_area (area) VALUES (221), (222);
10+
INSERT INTO iso_3166_1 (area, code) VALUES (221, 'GB'), (222, 'US');
11+
12+
INSERT INTO artist
13+
(id, gid, name, sort_name, type, gender, area,
14+
begin_area, end_area,
15+
begin_date_year, begin_date_month, begin_date_day,
16+
end_date_year, end_date_month, end_date_day, comment,
17+
last_updated)
18+
VALUES
19+
(3, '745c079d-374e-4436-9448-da92dedef3ce', 'Test Artist', 'Artist, Test', 1, 1, 221, 221, 221,
20+
2008, 01, 02, 2009, 03, 04, 'Yet Another Test Artist',
21+
'2009-07-09');
22+
23+
INSERT INTO artist (id, gid, name, sort_name) VALUES
24+
(4, '945c079d-374e-4436-9448-da92dedef3cf', 'Minimal Artist', 'Minimal Artist'),
25+
(5, 'dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958', 'Annotated Artist A', 'Annotated Artist A'),
26+
(6, 'ca4c2228-227c-4904-932a-dff442c091ea', 'Annotated Artist B', 'Annotated Artist B');
27+
28+
UPDATE artist_meta SET rating=70, rating_count=4 WHERE id=3;
29+
30+
INSERT INTO annotation (id, editor, text) VALUES (1, 1, 'Test annotation 1');
31+
INSERT INTO annotation (id, editor, text) VALUES (2, 1, 'Test annotation 2');
32+
INSERT INTO annotation (id, editor, text) VALUES (3, 1, 'Duplicate annotation');
33+
INSERT INTO annotation (id, editor, text) VALUES (4, 1, 'Duplicate annotation');
34+
35+
INSERT INTO artist_annotation (artist, annotation) VALUES (3, 1), (4, 2), (5, 3), (6, 4);
36+
37+
INSERT INTO artist_gid_redirect VALUES ('a4ef1d08-962e-4dd6-ae14-e42a6a97fc11', 3);

‎test/sql/cdstub.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
INSERT INTO release_raw (id, title, artist, added, last_modified, lookup_count, modify_count, source, barcode, comment)
2+
VALUES (1, 'Test Stub', 'Test Artist', '2000-01-01 0:00', '2001-01-01 0:00', 10, 1, 0, '837101029192', 'this is a comment');
3+
4+
INSERT INTO track_raw (release, title, artist, sequence)
5+
VALUES (1, 'Track title 1', '', 0);
6+
INSERT INTO track_raw (release, title, artist, sequence)
7+
VALUES (1, 'Track title 2', '', 1);
8+
9+
INSERT INTO cdtoc_raw (release, discid, track_count, leadout_offset, track_offset)
10+
VALUES (1, 'YfSgiOEayqN77Irs.VNV.UNJ0Zs-', 2, 20000, '{150,10000}');

‎test/sql/editor.sql‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
INSERT INTO editor (id, name, password, privs, email, website, bio, member_since,
2+
email_confirm_date, last_login_date, ha1)
3+
VALUES (1, 'new_editor', '{CLEARTEXT}password', 1+8+32+512, 'test@email.com', 'http://test.website',
4+
'biography', '1989-07-23', '2005-10-20', '2013-04-05', 'aa550c5b01407ef1f3f0d16daf9ec3c8'),
5+
(2, 'Alice', '{CLEARTEXT}secret1', 0, 'alice@example.com', 'http://example.com',
6+
'second biography', '2007-07-23', '2007-10-20', now(), 'e7f46e4f25ae38fcc952ef2b7edf0de9'),
7+
(3, 'kuno', '{CLEARTEXT}byld', 0, 'kuno@example.com', 'http://frob.nl',
8+
'donation check test user', '2010-03-25', '2010-03-25', now(), '00863261763ed5029ea051f87c4bbec3');
9+
10+
INSERT INTO editor_preference (editor, name, value)
11+
VALUES (1, 'datetime_format', '%m/%d/%Y %H:%M:%S'),
12+
(1, 'timezone', 'UTC'),
13+
(2, 'datetime_format', '%m/%d/%Y %H:%M:%S'),
14+
(2, 'timezone', 'UTC'),
15+
(2, 'public_ratings', '0'),
16+
(2, 'public_tags', '0');
17+
18+
INSERT INTO artist (id, gid, name, sort_name)
19+
VALUES (1, 'a9d99e40-72d7-11de-8a39-0800200c9a66', 'Name', 1);
20+
21+
INSERT INTO artist_credit (id, name, artist_count, gid)
22+
VALUES (1, 'Name', 1, '949a7fd5-fe73-3e8f-922e-01ff4ca958f7');
23+
INSERT INTO artist_credit_name (artist_credit, artist, name, position, join_phrase)
24+
VALUES (1, 1, 'Name', 0, '');
25+
26+
INSERT INTO release_group (id, gid, name, artist_credit)
27+
VALUES (1, '3b4faa80-72d9-11de-8a39-0800200c9a66', 'Arrival', 1);
28+
29+
INSERT INTO release (id, gid, name, artist_credit, release_group)
30+
VALUES (1, 'f34c079d-374e-4436-9448-da92dedef3ce', 'Arrival', 1, 1),
31+
(2, 'a34c079d-374e-4436-9448-da92dedef3ce', 'Arrival', 1, 1),
32+
(3, 'b34c079d-374e-4436-9448-da92dedef3ce', 'Arrival', 1, 1),
33+
(4, 'c34c079d-374e-4436-9448-da92dedef3ce', 'Arrival', 1, 1);
34+
35+
INSERT INTO editor_subscribe_editor (editor, subscribed_editor, last_edit_sent)
36+
VALUES (2, 1, 3);
37+
38+
INSERT INTO editor_collection (id, gid, editor, name, public, type)
39+
VALUES (1, 'f34c079d-374e-4436-9448-da92dedef3ce', 2, 'kunos collection', FALSE, 1),
40+
(2, 'd34c079d-374e-4436-9448-da92dedef3ce', 1, 'new_collection', TRUE, 1);
41+
42+
INSERT INTO editor_collection_release (collection, release)
43+
VALUES (1, 1), (1, 2);
44+
45+
INSERT INTO annotation (editor) VALUES (2); -- so Alice is not fully deleted
46+
47+
INSERT INTO old_editor_name (name) VALUES ('im_gone');

‎test/sql/event.sql‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
INSERT INTO area (id, gid, name, type)
2+
VALUES (3983, 'b9576171-3434-4d1b-8883-165ed6e65d2f', 'Kensington and Chelsea', 2)
3+
, (221, '8a754a16-0027-3a29-b6d7-2b40ea0481ed', 'United Kingdom', 1)
4+
, (38, '71bbafaa-e825-3e15-8ca9-017dcad1748b', 'Canada', 1);
5+
6+
INSERT INTO country_area (area) VALUES ( 38), (221);
7+
INSERT INTO iso_3166_1 (area, code) VALUES ( 38, 'CA'), (221, 'GB');
8+
9+
INSERT INTO place (id, gid, name, type, address, area, coordinates, begin_date_year)
10+
VALUES (729, '4352063b-a833-421b-a420-e7fb295dece0', 'Royal Albert Hall', 2, 'Kensington Gore, London SW7 2AP', 3983, '(51.50105,-0.17748)', 1871);
11+
12+
INSERT INTO event (id, gid, name, begin_date_year, begin_date_month, begin_date_day, end_date_year, end_date_month, end_date_day, time, type, cancelled, setlist, comment, ended)
13+
VALUES (59357, 'ca1d24c1-1999-46fd-8a95-3d4108df5cb2', 'BBC Open Music Prom', 2022, 9, 1, 2022, 9, 1, '19:30:00', 1, 'f', NULL, '2022, Prom 60', 't');
14+
15+
INSERT INTO artist (id, gid, name, sort_name, begin_date_year, begin_date_month, type, area, gender)
16+
VALUES (1294951, 'f72a5b32-449f-4090-9a2a-ebbdd8d3c2e5', 'Kwamé Ryan', 'Ryan, Kwamé', 1970, NULL, 1, 38, 1)
17+
, (831634, 'dfeba5ea-c967-4ad2-9cdd-3cffb4320143', 'BBC Concert Orchestra', 'BBC Concert Orchestr', 1952, 1, 5, 221, NULL);
18+
19+
INSERT INTO link (id, link_type) VALUES (199471, 794), (199854, 807), (199871, 806);
20+
21+
INSERT INTO l_event_place (id, link, entity0, entity1) VALUES (51345, 199471, 59357, 729);
22+
23+
INSERT INTO l_artist_event (id, link, entity0, entity1) VALUES (160762, 199854, 831634, 59357), (160763, 199871, 1294951, 59357);

‎test/sql/instrument.sql‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
INSERT INTO editor (id, name, password, privs, email, website, bio, member_since,
2+
email_confirm_date, last_login_date, ha1)
3+
VALUES (1, 'new_editor', '{CLEARTEXT}password', 1+8+32+512, 'test@email.com', 'http://test.website',
4+
'biography', '1989-07-23', '2005-10-20', '2013-04-05', 'aa550c5b01407ef1f3f0d16daf9ec3c8');
5+
6+
INSERT INTO instrument
7+
(id, gid, name, type, comment,
8+
description, last_updated)
9+
VALUES
10+
(3, '745c079d-374e-4436-9448-da92dedef3ce', 'Test Instrument', 2,
11+
'Yet Another Test Instrument', 'This is a description!', '2009-07-09');
12+
13+
INSERT INTO instrument (id, gid, name)
14+
VALUES (4, '945c079d-374e-4436-9448-da92dedef3cf', 'Minimal Instrument'),
15+
(5, 'a56d18ae-485f-5547-a559-eba3efef04d0', 'Minimal Instrument 2');
16+
17+
INSERT INTO artist (id, gid, name, sort_name) VALUES
18+
(1, '5441c29d-3602-4898-b1a1-b77fa23b8e50', 'David Bowie', 'David Bowie');
19+
20+
21+
INSERT INTO artist_credit (id, name, artist_count, gid)
22+
VALUES (1, 'David Bowie', 1, '949a7fd5-fe73-3e8f-922e-01ff4ca958f7');
23+
INSERT INTO artist_credit_name (artist_credit, position, artist, name) VALUES (1, 0, 1, 'David Bowie');
24+
25+
INSERT INTO recording (id, gid, name, artist_credit, length) VALUES
26+
(1, '722190f8-f718-482f-a8bc-a8d479426a30', '“Heroes”', 1, 370000);
27+
28+
INSERT INTO annotation (id, editor, text) VALUES (1, 1, 'Test annotation 1');
29+
INSERT INTO annotation (id, editor, text) VALUES (2, 1, 'Test annotation 2');
30+
31+
INSERT INTO instrument_annotation (instrument, annotation) VALUES (3, 1);
32+
INSERT INTO instrument_annotation (instrument, annotation) VALUES (4, 2);
33+
34+
INSERT INTO instrument_gid_redirect VALUES ('a4ef1d08-962e-4dd6-ae14-e42a6a97fc11', 3);
35+
36+
INSERT INTO link (id, link_type, attribute_count) VALUES (1, 148, 2);
37+
INSERT INTO link_attribute (link, attribute_type) VALUES (1, (SELECT id FROM link_attribute_type WHERE gid = '945c079d-374e-4436-9448-da92dedef3cf'));
38+
INSERT INTO link_attribute_credit (link, attribute_type, credited_as) VALUES (1, (SELECT id FROM link_attribute_type WHERE gid = '945c079d-374e-4436-9448-da92dedef3cf'), 'blah instrument');
39+
INSERT INTO link_attribute (link, attribute_type) VALUES (1, (SELECT id FROM link_attribute_type WHERE gid = 'a56d18ae-485f-5547-a559-eba3efef04d0'));
40+
INSERT INTO link_attribute_credit (link, attribute_type, credited_as) VALUES (1, (SELECT id FROM link_attribute_type WHERE gid = 'a56d18ae-485f-5547-a559-eba3efef04d0'), 'stupid instrument');
41+
INSERT INTO l_artist_recording (id, link, entity0, entity1) VALUES (4, 1, 1, 1);

‎test/sql/label.sql‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
INSERT INTO area (id, gid, name, type) VALUES
2+
(221, '8a754a16-0027-3a29-b6d7-2b40ea0481ed', 'United Kingdom', 1);
3+
INSERT INTO country_area (area) VALUES (221);
4+
INSERT INTO iso_3166_1 (area, code) VALUES (221, 'GB');
5+
6+
INSERT INTO label (id, gid, name, type, area, label_code,
7+
begin_date_year, begin_date_month, begin_date_day,
8+
end_date_year, end_date_month, end_date_day, comment)
9+
VALUES (3, '46f0f4cd-8aab-4b33-b698-f459faf64190', 'Warp Records', 3, 221, 2070,
10+
1989, 02, 03, 2008, 05, 19, 'Sheffield based electronica label');
11+
12+
INSERT INTO label (id, gid, name)
13+
VALUES (2, 'f2a9a3c0-72e3-11de-8a39-0800200c9a66', 'To Merge');
14+
15+
INSERT INTO editor (id, name, password, ha1) VALUES (1, 'editor', '{CLEARTEXT}pass', '3f3edade87115ce351d63f42d92a1834');
16+
INSERT INTO annotation (id, editor, text, changelog) VALUES (1, 1, 'Label Annotation', 'Changes');
17+
INSERT INTO label_annotation (label, annotation) VALUES (3, 1);
18+
19+
INSERT INTO label_gid_redirect (gid, new_id) VALUES ('efdf3fe9-c293-4acd-b4b2-8d2a7d4f9592', 3);

0 commit comments

Comments
 (0)