[Metakit] memory leak?

[zooko at zooko.com]

I think my most recent message might have been unclear.  The simple script
included in my earlier note appears to demonstrate a memory leak, even with
regular calls to store.commit().  (NB: not "db.commit()").

Here is a more complicated script that attempts to purge from memory any
leftover state.  (It also prints out its current memory usage as it runs.) It,
too, continually leaks memory until it uses up all system memory and is killed.

Regards,

Zooko

-------

import gc, os, random, sys, time

import metakit
print metakit, metakit.__version__

def get_mem_usage():
    """
    This only works on Linux, and only if the /proc/$PID/stat output is the
    same as that in linux kernel 2.4.  Also `os.getpid()' must work.

    @returns the virtual memory used by this process
    """
    # sample output from cat /proc/$PID/stat:
    # 19606 (python2.2) S 19592 19592 7451 34834 19592 0 2155 0 970 0 57 12 0 0 9 0 0 0 183707687 36585472 5206 4294967295 134512640 135126528 3221221280 3221219220 1074927566 0 0 4096 137912326 3222525185 0 0 17 0
    a = os.popen("cat /proc/%s/stat" % os.getpid()).read().split()
    assert a[0] == "%s" % os.getpid()
    return int(a[22])


store = None
db = None

USE_HASH_VIEW=False

def init_mk():
    global store, db
    store = metakit.storage("whee", 1)
    if USE_HASH_VIEW:
        _FORMAT = "test_dbB[uniqid:B,thing:I]"
        hashvw = store.getas("test_hash[_H:I,_R:I]")
        db = store.getas(_FORMAT).hash(hashvw, 1)
    else:
        _FORMAT = "test_db[_B[uniqid:B,thing:I]]"
        # else, use blocked, ordered view
        db = store.getas(_FORMAT).blocked().ordered()
    
def insert_mk(N):
    for i in range(N):
        uniqid = os.urandom(20)
        thing = random.randrange(0, 2**20)
        dbhit = db.find(uniqid=uniqid)
        assert dbhit == -1
        db.insert(dbhit, uniqid=uniqid, thing=thing)
    store.commit()
    gc.collect()

def teardown_mk():
    global db, store
    db = None
    store.commit()
    store = None
    gc.collect()
    os.remove("whee")

while True:
    init_mk()
    print "before -- % 10.f bytes" % get_mem_usage(), 
    insert_mk(2**18)
    print "after -- % 10.f bytes" % get_mem_usage(), 
    teardown_mk()
    print "clean -- % 10.f bytes" % get_mem_usage()

-------------- next part --------------
#!/usr/bin/env python

import gc, os, random, sys, time

import metakit
print metakit, metakit.__version__

def get_mem_usage():
    """
    This only works on Linux, and only if the /proc/$PID/stat output is the
    same as that in linux kernel 2.4.  Also `os.getpid()' must work.

    @returns the virtual memory used by this process
    """
    # sample output from cat /proc/$PID/stat:
    # 19606 (python2.2) S 19592 19592 7451 34834 19592 0 2155 0 970 0 57 12 0 0 9 0 0 0 183707687 36585472 5206 4294967295 134512640 135126528 3221221280 3221219220 1074927566 0 0 4096 137912326 3222525185 0 0 17 0
    a = os.popen("cat /proc/%s/stat" % os.getpid()).read().split()
    assert a[0] == "%s" % os.getpid()
    return int(a[22])


store = None
db = None

USE_HASH_VIEW=False

def init_mk():
    global store, db
    store = metakit.storage("whee", 1)
    if USE_HASH_VIEW:
        _FORMAT = "test_dbB[uniqid:B,thing:I]"
        hashvw = store.getas("test_hash[_H:I,_R:I]")
        db = store.getas(_FORMAT).hash(hashvw, 1)
    else:
        _FORMAT = "test_db[_B[uniqid:B,thing:I]]"
        # else, use blocked, ordered view
        db = store.getas(_FORMAT).blocked().ordered()
    
def insert_mk(N):
    for i in range(N):
        uniqid = os.urandom(20)
        thing = random.randrange(0, 2**20)
        dbhit = db.find(uniqid=uniqid)
        assert dbhit == -1
        db.insert(dbhit, uniqid=uniqid, thing=thing)
    store.commit()
    gc.collect()

def teardown_mk():
    global db, store
    db = None
    store.commit()
    store = None
    gc.collect()
    os.remove("whee")

while True:
    init_mk()
    print "before -- % 10.f bytes" % get_mem_usage(), 
    insert_mk(2**18)
    print "after -- % 10.f bytes" % get_mem_usage(), 
    teardown_mk()
    print "clean -- % 10.f bytes" % get_mem_usage()