Skip to content

Commit a925617

Browse files
elpranselprans
committed
Fix type introspection being very slow on large databases
Stop using CTEs in the type introspection query, otherwise it runs for ages on databases with a large number of composite attributes (i. e. tons of tables with tons of columns). Fixes: #186
1 parent 23394c9 commit a925617

File tree

3 files changed

+119
-70
lines changed

3 files changed

+119
-70
lines changed

‎asyncpg/_testbase.py

+9-6
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,12 @@ def assertRunUnder(self, delta):
9393
try:
9494
yield
9595
finally:
96-
if time.monotonic() - st > delta:
96+
elapsed = time.monotonic() - st
97+
if elapsed > delta:
9798
raise AssertionError(
98-
'running block took longer than {}'.format(delta))
99+
'running block took {:0.3f}s which is longer '
100+
'than the expected maximum of {:0.3f}s'.format(
101+
elapsed, delta))
99102

100103
@contextlib.contextmanager
101104
def assertLoopErrorHandlerCalled(self, msg_re: str):
@@ -214,18 +217,18 @@ def wrap(func):
214217

215218
class ConnectedTestCase(ClusterTestCase):
216219

217-
def getExtraConnectOptions(self):
218-
return {}
219-
220220
def setUp(self):
221221
super().setUp()
222222

223223
# Extract options set up with `with_connection_options`.
224224
test_func = getattr(self, self._testMethodName).__func__
225225
opts = getattr(test_func, '__connect_options__', {})
226+
if 'database' not in opts:
227+
opts = dict(opts)
228+
opts['database'] = 'postgres'
226229

227230
self.con = self.loop.run_until_complete(
228-
self.cluster.connect(database='postgres', loop=self.loop, **opts))
231+
self.cluster.connect(loop=self.loop, **opts))
229232

230233
self.server_version = self.con.get_server_version()
231234

‎asyncpg/introspection.py

+60-64
Original file line numberDiff line numberDiff line change
@@ -5,29 +5,8 @@
55
# the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0
66

77

8-
INTRO_LOOKUP_TYPES = '''\
9-
WITH RECURSIVE typeinfo_tree(
10-
oid, ns, name, kind, basetype, has_bin_io, elemtype, elemdelim,
11-
range_subtype, elem_has_bin_io, attrtypoids, attrnames, depth)
12-
AS (
13-
WITH composite_attrs
14-
AS (
15-
SELECT
16-
c.reltype AS comptype_oid,
17-
array_agg(ia.atttypid ORDER BY ia.attnum) AS typoids,
18-
array_agg(ia.attname::text ORDER BY ia.attnum) AS names
19-
FROM
20-
pg_attribute ia
21-
INNER JOIN pg_class c
22-
ON (ia.attrelid = c.oid)
23-
WHERE
24-
ia.attnum > 0 AND NOT ia.attisdropped
25-
GROUP BY
26-
c.reltype
27-
),
28-
29-
typeinfo
30-
AS (
8+
_TYPEINFO = '''\
9+
(
3110
SELECT
3211
t.oid AS oid,
3312
ns.nspname AS ns,
@@ -76,16 +55,28 @@
7655
elem_t.typsend::oid != 0
7756
END) AS elem_has_bin_io,
7857
(CASE WHEN t.typtype = 'c' THEN
79-
(SELECT ca.typoids
80-
FROM composite_attrs AS ca
81-
WHERE ca.comptype_oid = t.oid)
58+
(SELECT
59+
array_agg(ia.atttypid ORDER BY ia.attnum)
60+
FROM
61+
pg_attribute ia
62+
INNER JOIN pg_class c
63+
ON (ia.attrelid = c.oid)
64+
WHERE
65+
ia.attnum > 0 AND NOT ia.attisdropped
66+
AND c.reltype = t.oid)
8267
8368
ELSE NULL
8469
END) AS attrtypoids,
8570
(CASE WHEN t.typtype = 'c' THEN
86-
(SELECT ca.names
87-
FROM composite_attrs AS ca
88-
WHERE ca.comptype_oid = t.oid)
71+
(SELECT
72+
array_agg(ia.attname::text ORDER BY ia.attnum)
73+
FROM
74+
pg_attribute ia
75+
INNER JOIN pg_class c
76+
ON (ia.attrelid = c.oid)
77+
WHERE
78+
ia.attnum > 0 AND NOT ia.attisdropped
79+
AND c.reltype = t.oid)
8980
9081
ELSE NULL
9182
END) AS attrnames
@@ -102,13 +93,20 @@
10293
t.oid = range_t.rngtypid
10394
)
10495
)
96+
'''
97+
10598

99+
INTRO_LOOKUP_TYPES = '''\
100+
WITH RECURSIVE typeinfo_tree(
101+
oid, ns, name, kind, basetype, has_bin_io, elemtype, elemdelim,
102+
range_subtype, elem_has_bin_io, attrtypoids, attrnames, depth)
103+
AS (
106104
SELECT
107105
ti.oid, ti.ns, ti.name, ti.kind, ti.basetype, ti.has_bin_io,
108106
ti.elemtype, ti.elemdelim, ti.range_subtype, ti.elem_has_bin_io,
109107
ti.attrtypoids, ti.attrnames, 0
110108
FROM
111-
typeinfo AS ti
109+
{typeinfo} AS ti
112110
WHERE
113111
ti.oid = any($1::oid[])
114112
@@ -119,7 +117,7 @@
119117
ti.elemtype, ti.elemdelim, ti.range_subtype, ti.elem_has_bin_io,
120118
ti.attrtypoids, ti.attrnames, tt.depth + 1
121119
FROM
122-
typeinfo ti,
120+
{typeinfo} ti,
123121
typeinfo_tree tt
124122
WHERE
125123
(tt.elemtype IS NOT NULL AND ti.oid = tt.elemtype)
@@ -133,33 +131,12 @@
133131
typeinfo_tree
134132
ORDER BY
135133
depth DESC
136-
'''
134+
'''.format(typeinfo=_TYPEINFO)
137135

138136

139137
# Prior to 9.2 PostgreSQL did not have range types.
140-
INTRO_LOOKUP_TYPES_91 = '''\
141-
WITH RECURSIVE typeinfo_tree(
142-
oid, ns, name, kind, basetype, has_bin_io, elemtype, elemdelim,
143-
range_subtype, elem_has_bin_io, attrtypoids, attrnames, depth)
144-
AS (
145-
WITH composite_attrs
146-
AS (
147-
SELECT
148-
c.reltype AS comptype_oid,
149-
array_agg(ia.atttypid ORDER BY ia.attnum) AS typoids,
150-
array_agg(ia.attname::text ORDER BY ia.attnum) AS names
151-
FROM
152-
pg_attribute ia
153-
INNER JOIN pg_class c
154-
ON (ia.attrelid = c.oid)
155-
WHERE
156-
ia.attnum > 0 AND NOT ia.attisdropped
157-
GROUP BY
158-
c.reltype
159-
),
160-
161-
typeinfo
162-
AS (
138+
_TYPEINFO_91 = '''\
139+
(
163140
SELECT
164141
t.oid AS oid,
165142
ns.nspname AS ns,
@@ -199,16 +176,28 @@
199176
elem_t.typsend::oid != 0
200177
AS elem_has_bin_io,
201178
(CASE WHEN t.typtype = 'c' THEN
202-
(SELECT ca.typoids
203-
FROM composite_attrs AS ca
204-
WHERE ca.comptype_oid = t.oid)
179+
(SELECT
180+
array_agg(ia.atttypid ORDER BY ia.attnum)
181+
FROM
182+
pg_attribute ia
183+
INNER JOIN pg_class c
184+
ON (ia.attrelid = c.oid)
185+
WHERE
186+
ia.attnum > 0 AND NOT ia.attisdropped
187+
AND c.reltype = t.oid)
205188
206189
ELSE NULL
207190
END) AS attrtypoids,
208191
(CASE WHEN t.typtype = 'c' THEN
209-
(SELECT ca.names
210-
FROM composite_attrs AS ca
211-
WHERE ca.comptype_oid = t.oid)
192+
(SELECT
193+
array_agg(ia.attname::text ORDER BY ia.attnum)
194+
FROM
195+
pg_attribute ia
196+
INNER JOIN pg_class c
197+
ON (ia.attrelid = c.oid)
198+
WHERE
199+
ia.attnum > 0 AND NOT ia.attisdropped
200+
AND c.reltype = t.oid)
212201
213202
ELSE NULL
214203
END) AS attrnames
@@ -222,13 +211,20 @@
222211
t.typelem = elem_t.oid
223212
)
224213
)
214+
'''
215+
216+
INTRO_LOOKUP_TYPES_91 = '''\
217+
WITH RECURSIVE typeinfo_tree(
218+
oid, ns, name, kind, basetype, has_bin_io, elemtype, elemdelim,
219+
range_subtype, elem_has_bin_io, attrtypoids, attrnames, depth)
220+
AS (
225221
226222
SELECT
227223
ti.oid, ti.ns, ti.name, ti.kind, ti.basetype, ti.has_bin_io,
228224
ti.elemtype, ti.elemdelim, ti.range_subtype, ti.elem_has_bin_io,
229225
ti.attrtypoids, ti.attrnames, 0
230226
FROM
231-
typeinfo AS ti
227+
{typeinfo} AS ti
232228
WHERE
233229
ti.oid = any($1::oid[])
234230
@@ -239,7 +235,7 @@
239235
ti.elemtype, ti.elemdelim, ti.range_subtype, ti.elem_has_bin_io,
240236
ti.attrtypoids, ti.attrnames, tt.depth + 1
241237
FROM
242-
typeinfo ti,
238+
{typeinfo} ti,
243239
typeinfo_tree tt
244240
WHERE
245241
(tt.elemtype IS NOT NULL AND ti.oid = tt.elemtype)
@@ -253,7 +249,7 @@
253249
typeinfo_tree
254250
ORDER BY
255251
depth DESC
256-
'''
252+
'''.format(typeinfo=_TYPEINFO_91)
257253

258254

259255
TYPE_BY_NAME = '''\

‎tests/test_introspection.py

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Copyright (C) 2016-present the asyncpg authors and contributors
2+
# <see AUTHORS file>
3+
#
4+
# This module is part of asyncpg and is released under
5+
# the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0
6+
7+
8+
from asyncpg import _testbase as tb
9+
10+
11+
MAX_RUNTIME = 0.1
12+
13+
14+
class TestTimeout(tb.ConnectedTestCase):
15+
@classmethod
16+
def setUpClass(cls):
17+
super().setUpClass()
18+
cls.adminconn = cls.loop.run_until_complete(
19+
cls.cluster.connect(database='postgres', loop=cls.loop))
20+
cls.loop.run_until_complete(
21+
cls.adminconn.execute('CREATE DATABASE asyncpg_intro_test'))
22+
23+
@classmethod
24+
def tearDownClass(cls):
25+
cls.loop.run_until_complete(
26+
cls.adminconn.execute('DROP DATABASE asyncpg_intro_test'))
27+
28+
cls.loop.run_until_complete(cls.adminconn.close())
29+
cls.adminconn = None
30+
31+
super().tearDownClass()
32+
33+
@tb.with_connection_options(database='asyncpg_intro_test')
34+
async def test_introspection_on_large_db(self):
35+
await self.con.execute(
36+
'CREATE DOMAIN intro_test AS int'
37+
)
38+
39+
await self.con.execute(
40+
'CREATE TABLE base ({})'.format(
41+
','.join('c{:02} varchar'.format(n) for n in range(50))
42+
)
43+
)
44+
for n in range(1000):
45+
await self.con.execute(
46+
'CREATE TABLE child_{:04} () inherits (base)'.format(n)
47+
)
48+
49+
with self.assertRunUnder(MAX_RUNTIME):
50+
await self.con.fetchval('SELECT $1::intro_test', 1)

0 commit comments

Comments
 (0)