[svn] r3153 - in trunk/tools/pydar2: pydar sql

packagers at lists.rpmforge.net packagers at lists.rpmforge.net
Thu Apr 21 21:28:47 CEST 2005


Author: dries
Date: 2005-04-21 21:28:39 +0200 (Thu, 21 Apr 2005)
New Revision: 3153

Added:
   trunk/tools/pydar2/pydar/filter.py
   trunk/tools/pydar2/pydar/filterchain.py
   trunk/tools/pydar2/pydar/norpmforgetemplatesfilter.py
Modified:
   trunk/tools/pydar2/pydar/config.py
   trunk/tools/pydar2/pydar/postgresqlstorage.py
   trunk/tools/pydar2/pydar/specrepository.py
   trunk/tools/pydar2/pydar/specrepositoryspecfile.py
   trunk/tools/pydar2/pydar/storage.py
   trunk/tools/pydar2/pydar/svncheck.py
   trunk/tools/pydar2/sql/svncheck.sql
Log:
update

Modified: trunk/tools/pydar2/pydar/config.py
===================================================================
--- trunk/tools/pydar2/pydar/config.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/config.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -23,6 +23,7 @@
 from specrepository import SpecRepository
 from specrepositoryfactory import SpecRepositoryFactory
 from specrepositorylist import SpecRepositoryList
+from filter import Filter
 from log4py import Logger
 
 class Config:
@@ -91,7 +92,8 @@
         self.dirtestbuilds = myParser.get('slave','dirtestbuilds')
 
         self.__savespecrepositoryspecfiletags = True
-
+        #self.__
+        
         self.sleeptime = 5
 
         # get the supported machroots
@@ -110,15 +112,17 @@
         #sr = SpecRepository("rpmforge",SpecRepository.TYPE_SVN,self.specreposrootdir + "/rpmforge/rpms")
         srf = SpecRepositoryFactory.getInstance();
         sr = srf.createSpecRepository("rpmforge",SpecRepository.TYPE_SVN,self.__specreposrootdir + "/rpmforge/rpms")
+        # we add a filter so the spec templates used by rpmforge are not stored in the database
+        sr.appendSpecRepositorySpecFileFilter(Filter("pydar/norpmforgetemplatesfilter","NoRpmforgeTemplatesFilter"))
         self.getSpecRepositoryList().add(sr)
         #sr = SpecRepository("pld",SpecRepository.TYPE_CVS,self.specreposrootdir + "/pld/SPECS")
-        sr = srf.createSpecRepository("pld",SpecRepository.TYPE_CVS,self.__specreposrootdir + "/pld/SPECS")
-        self.getSpecRepositoryList().add(sr)
-        #sr = SpecRepository("fedoraextrasdevel",SpecRepository.TYPE_CVS,self.specreposrootdir + "/fedoraextrasdevel")
-        #self.specrepositories.append(sr)
+        #sr = srf.createSpecRepository("pld",SpecRepository.TYPE_CVS,self.__specreposrootdir + "/pld/SPECS")
+        #self.getSpecRepositoryList().add(sr)
+        # #sr = SpecRepository("fedoraextrasdevel",SpecRepository.TYPE_CVS,self.specreposrootdir + "/fedoraextrasdevel")
+        # #self.specrepositories.append(sr)
         #sr = SpecRepository("local",SpecRepositoryFactory.TYPE_FILES,self.specreposrootdir + "/local")
-        sr = srf.createSpecRepository("local",SpecRepository.TYPE_FILES,self.__specreposrootdir + "/local")
-        self.getSpecRepositoryList().add(sr)
+        #sr = srf.createSpecRepository("local",SpecRepository.TYPE_FILES,self.__specreposrootdir + "/local")
+        #self.getSpecRepositoryList().add(sr)
  
         __withinFullInit = False
         __fullInitDone = True

Added: trunk/tools/pydar2/pydar/filter.py
===================================================================
--- trunk/tools/pydar2/pydar/filter.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/filter.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+
+""" A filter: a wrapper around a filter function in a seperate .py file """
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2004 Dries Verachtert
+
+import sys
+from log4py import Logger
+
+class Filter:
+    def __init__(self, modulePath, functionName):
+        self.__cat = Logger().get_instance(self)
+        # immediately try to create the callable object
+        sys.path.append(".")
+        # @todo: add some nice exception catching so we can show decent errormessages
+        myModule = __import__(modulePath)
+        myMethod = getattr(myModule,functionName)
+        if not callable(myMethod):
+            raise Exception("method " + functionName + " in module " + modulePath + " is not callable!")
+        self.__myMethod = myMethod
+        
+        
+    def accepts(self,value):
+        return self.__myMethod.__call__(value)

Added: trunk/tools/pydar2/pydar/filterchain.py
===================================================================
--- trunk/tools/pydar2/pydar/filterchain.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/filterchain.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -0,0 +1,35 @@
+#!/usr/bin/python
+
+""" contains a chain of filters """
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2004 Dries Verachtert
+
+from log4py import Logger
+
+class FilterChain:
+    
+    def __init__(self):
+        self.__cat = Logger().get_instance(self)
+        self.__filterList = []
+            
+    def appendFilter(self, newFilter):
+        self.__filterList.append(newFilter)
+        
+    def accepts(self, value):
+        for f in self.__filterList:
+            if not f.accepts(value):
+                return False
+        return True

Added: trunk/tools/pydar2/pydar/norpmforgetemplatesfilter.py
===================================================================
--- trunk/tools/pydar2/pydar/norpmforgetemplatesfilter.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/norpmforgetemplatesfilter.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -0,0 +1,30 @@
+#!/usr/bin/python
+
+""" Small example filter: make sure the templates of spec files are not stored in the database """
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2004 Dries Verachtert
+
+# The rpmforge subversion also contains some templates of spec files and we do not want them in 
+# the database. This filter only accepts SpecRepositorySpecFile objects which do not match 
+# any of the tempate spec files.
+
+def NoRpmforgeTemplatesFilter(specRepoSpecFile):
+    print "called with obj " + str(specRepoSpecFile)
+    fn = specRepoSpecFile.getFileName()
+    if fn == "_template.spec" or fn == "_template-fr.spec" or fn == "_perl-template.spec":
+        return False
+    else:
+        return True

Modified: trunk/tools/pydar2/pydar/postgresqlstorage.py
===================================================================
--- trunk/tools/pydar2/pydar/postgresqlstorage.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/postgresqlstorage.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -89,8 +89,10 @@
     
     def __isSpecRepositorySpecFileVersionInfoPresent(self,specfileid,currentversion):
         conn = self.__getConnection()
-        cursor = self.dbconnection.cursor()
-        cursor.execute("select * from pydar2_specfile_version where specfileid=" + specfileid + " and version=" + currentversion)
+        cursor = conn.cursor()
+        sql = "select version from pydar2_specfile_version where specfileid=" + str(specfileid) + " and version=" + str(currentversion)
+        self.__cat.debug(sql)
+        cursor.execute(sql)
         retval = 0
         if cursor.rowcount > 0: 
             retval = 1
@@ -101,7 +103,7 @@
     def __insertSpecRepositorySpecfileVersion(self,specfileid,currentversion):
         conn = self.__getConnection()
         cursor = conn.cursor()
-        cursor.execute("insert into pydar2_specfile_version (specfileid,version) values (" + specfileid + "," + version + ")")
+        cursor.execute("insert into pydar2_specfile_version (specfileid,version) values (" + str(specfileid) + "," + str(currentversion) + ")")
         conn.commit()
         cursor.close()
         self.__releaseConnection(conn)
@@ -110,7 +112,7 @@
         self.__cat.debug("start")
         specfileid = specRepoSpecFile.getId()
         currentversion = specRepoSpecFile.getVersion()
-        self.__cat.debug("current version is " + currentversion)
+        self.__cat.debug("current version is " + str(currentversion))
         alreadyImported = self.__isSpecRepositorySpecFileVersionInfoPresent(specfileid,currentversion)
         if alreadyImported:
             # strange.. this function is normally only called when the file is changed
@@ -119,16 +121,64 @@
         else:
             self.__cat.debug("need to import all the variables..")
             self.__insertSpecRepositorySpecfileVersion(specfileid,currentversion)
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'fc3', specDir,fullRpmDir,fileName, ' --define "fc3 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'fc2', specDir,fullRpmDir,fileName, ' --define "fc2 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'fc1', specDir,fullRpmDir,fileName, ' --define "fc1 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh9', specDir,fullRpmDir,fileName, ' --define "rh9 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh8', specDir,fullRpmDir,fileName, ' --define "rh8 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh7', specDir,fullRpmDir,fileName, ' --define "rh7 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh6', specDir,fullRpmDir,fileName, ' --define "rh6 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'el3', specDir,fullRpmDir,fileName, ' --define "el3 1"')
-            self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'el2', specDir,fullRpmDir,fileName, ' --define "el2 1"') 
+            self.__insertSpecRepositorySpecFileTagsPerDistro(specRepo, specRepoSpecFile, currentversion)
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'fc3', specDir,fullRpmDir,fileName, ' --define "fc3 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'fc2', specDir,fullRpmDir,fileName, ' --define "fc2 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'fc1', specDir,fullRpmDir,fileName, ' --define "fc1 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh9', specDir,fullRpmDir,fileName, ' --define "rh9 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh8', specDir,fullRpmDir,fileName, ' --define "rh8 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh7', specDir,fullRpmDir,fileName, ' --define "rh7 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'rh6', specDir,fullRpmDir,fileName, ' --define "rh6 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'el3', specDir,fullRpmDir,fileName, ' --define "el3 1"')
+            #self.__insertSpecRepositorySpecFileTags(specfileid, currentversion, 'el2', specDir,fullRpmDir,fileName, ' --define "el2 1"') 
     
+    # returns a hash wich contains the fields    code => defines  from the db table  pydar2_distro
+    def __getDistroCodeToDefine(self):
+        retval = {}
+        conn = self.__getConnection()
+        cursor = conn.cursor()
+        sql = "select code, defines from pydar2_distro where disabled='0'"
+        cursor.execute(sql)
+        rsdistros = cursor.fetchall()
+        for row in rsdistros:
+            code = row[0]
+            defines = row[1]
+            print "code=" + code
+            print "defines=" + defines
+            retval[code] = defines
+        cursor.close()
+        self.__releaseConnection(conn)
+        return retval
+    
+    def __insertSpecRepositorySpecFileTagsPerDistro(self,specRepo, specRepoSpecFile,currentversion):
+        specfileid= specRepoSpecFile.getId()
+        self.__cat.debug("start,specfileid=" + str(specRepoSpecFile.getId()) + ",currentversion=" + str(currentversion))
+        distros = self.__getDistroCodeToDefine()
+        conn = self.__getConnection()
+        cursor = conn.cursor()
+        for distrocode in distros.keys():
+            defines = distros[distrocode]
+            sf = specRepoSpecFile.getSpecFileTags(defines)
+            
+            for tag in sf.rpmforgetags:
+                print 'rpmforge tag: ' + tag
+                name = tag
+                #print 'value: ' + sf.rpmforgetags[tag]
+                value = string.replace(sf.rpmforgetags[tag],"'","''")
+                value = string.rstrip(value,"\\")
+                cursor.execute("insert into pydar2_specfile_tags (specfileid,version,distroid,name,value) values (" + str(specfileid) + "," + str(currentversion) + ",(select id from pydar2_distro where code='" + str(distrocode) + "'),'"+ str(name) + "','" + str(value) + "')")
+            for tag in sf.standardtags:
+                print 'standard tag: ' + tag
+                name = tag
+                # print 'value: ' + string.replace(sf.standardtags[tag],"'","''")
+                value = string.replace(sf.standardtags[tag],"'","''")
+                value = string.rstrip(value,"\\")
+                cursor.execute("insert into pydar2_specfile_tags (specfileid,version,distroid,name,value) values (" + str(specfileid) + "," + str(currentversion) + ",(select id from pydar2_distro where code='" + str(distrocode) + "'),'"+ str(name) + "','" + str(value) + "')")
+        
+        conn.commit()
+        cursor.close()
+        self.__releaseConnection(conn)
+    
     def saveSpecRepository(self,specRepo):
         conn = self.__getConnection()
         cursor = conn.cursor()

Modified: trunk/tools/pydar2/pydar/specrepository.py
===================================================================
--- trunk/tools/pydar2/pydar/specrepository.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/specrepository.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -20,6 +20,7 @@
 from log4py import Logger
 from storagefactory import StorageFactory
 from specrepositoryspecfile import SpecRepositorySpecFile
+from filterchain import FilterChain
 import posixpath, posix, string
 
 # A specrepository has some attributes (a name, the root directory)
@@ -44,11 +45,15 @@
         self.__specFiles = [];
         self.__storage = StorageFactory.getStorage()
         self.__id = None
+        self.__filterChain = FilterChain()
         
     # get a unique identifier for the current version of this file
     # has to be implemented by a subclass
     def getSpecFileVersion(self,specRepoSpecFile):
         raise Exception("getSpecFileVersion mus be implemented by a subclass")
+
+    def appendSpecRepositorySpecFileFilter(self,newFilter):
+        self.__filterChain.appendFilter(newFilter)
         
     def getUpdateCommand(self):
         return ""
@@ -144,5 +149,6 @@
                 if rindex > 0 and rindex == len(entryInDir) - len('.spec'):
                     #self.cat.debug("spec match, entryInDir=" + entryInDir)
                     spf = SpecRepositorySpecFile(self,startDir,entryInDir)
-                    arr.append(spf)
+                    if self.__filterChain.accepts(spf):
+                        arr.append(spf)
         return arr

Modified: trunk/tools/pydar2/pydar/specrepositoryspecfile.py
===================================================================
--- trunk/tools/pydar2/pydar/specrepositoryspecfile.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/specrepositoryspecfile.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -20,6 +20,7 @@
 # also calculates the SHA1 checksum so it is easy to check if a file is changed or not
 
 from log4py import Logger
+from specfiletags import SpecFileTags
 import sha
 
 class SpecRepositorySpecFile:
@@ -38,9 +39,13 @@
     # each version of a spec file should have a unique version
     # the version is defined by the SpecRepository
     def getVersion(self):
-        self.__specRepository.getSpecFileVersion(self)
+        return self.__specRepository.getSpecFileVersion(self)
+    
+    # construct an object which contains all the tags within this spec file
+    # definetags is something like   --define el4 1        or   --define fc2 1 --define au1.92 1
+    def getSpecFileTags(self, definetags):
+        return SpecFileTags(self.getSpecRepository().getRootDirectory() + "/" + self.getPathFromRoot(), definetags)
         
-        
     def __calculateCheckSum(self):
         s = sha.new()
         fullFileName = self.__specRepository.getRootDirectory() + "/" + self.__subDir + "/" + self.__fileName

Modified: trunk/tools/pydar2/pydar/storage.py
===================================================================
--- trunk/tools/pydar2/pydar/storage.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/storage.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -41,7 +41,7 @@
         #raise Exception("todo getConfig")
         if cls.__myConfig == None:
             cls.__myConfig = config.Config.getInstance()
-            return cls.__myConfig
+        return cls.__myConfig
     getConfig = classmethod(getConfig)
     
     ## classmethod

Modified: trunk/tools/pydar2/pydar/svncheck.py
===================================================================
--- trunk/tools/pydar2/pydar/svncheck.py	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/pydar/svncheck.py	2005-04-21 19:28:39 UTC (rev 3153)
@@ -106,6 +106,7 @@
         cursor.close()
         return retval
             
+    # moved to __insertSpecRepositorySpecFileTagsPerDistro in PostgresqlStorage
     def insertTags(self,specfileid,svncmtrev,distrocode,specDir,fullRpmDir,fileName,deftags):
         self.cat.debug("start,specfileid=" + specfileid + ",svncmtrev=" + svncmtrev + ",distrocode=" + distrocode)
         sf = SpecFile(fullRpmDir+"/"+fileName,deftags)
@@ -161,4 +162,4 @@
             self.dbconnection.commit()
             cursor.close()
             return self.getCurrentOrNewDbSpecFileId(specDir,fullRpmDir,fileName)
-        
\ No newline at end of file
+        

Modified: trunk/tools/pydar2/sql/svncheck.sql
===================================================================
--- trunk/tools/pydar2/sql/svncheck.sql	2005-04-21 14:30:11 UTC (rev 3152)
+++ trunk/tools/pydar2/sql/svncheck.sql	2005-04-21 19:28:39 UTC (rev 3153)
@@ -37,19 +37,20 @@
 
 create table pydar2_specfile_version (
 	specfileid	integer not null,
-	svncmtrev	integer not null,
+	version 	integer not null,
         addtimestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 create index pydar2_specfile_v__sfid on pydar2_specfile_version(specfileid);
-create index pydar2_specfile_v__scr on pydar2_specfile_version(svncmtrev);
-create index pydar2_specfile_v__both on pydar2_specfile_version(specfileid,svncmtrev);
+create index pydar2_specfile_v__scr on pydar2_specfile_version(version);
+create index pydar2_specfile_v__both on pydar2_specfile_version(specfileid,version);
 
 
 create table pydar2_distro (
     id          integer unique not null default nextval('pydar2_specfile_idseq'),
     name        varchar(512) unique not null,
     code        varchar(10) unique not null,
-	defines		varchar(512) not null,
+    defines     varchar(512) not null,
+    disabled    boolean not null default '0'
 );  
 
 insert into pydar2_distro (name,code,defines) values ('Fedora Core 3','fc3',' --define "fc3 1"');
@@ -72,7 +73,7 @@
 	value		varchar(4000) not null
 );
 create index pydar2_specfile_t__sfid on pydar2_specfile_tags(specfileid);
-create index pydar2_specfile_t__scr on pydar2_specfile_tags(svncmtrev);
+create index pydar2_specfile_t__scr on pydar2_specfile_tags(version);
 create index pydar2_specfile_t__did on pydar2_specfile_tags(distroid);
 create index pydar2_specfile_t__both on pydar2_specfile_tags(specfileid,version);
 create index pydar2_specfile_t__three on pydar2_specfile_tags(specfileid,version,distroid);
@@ -83,11 +84,11 @@
 create index pydar2_specfile_t_value on pydar2_specfile_tags(value);
 
 
-create VIEW pydar2_specfile_lastversion AS select max(v.version) as latestversion, v.specfileid, s.name 
-from pydar2_specfile_version v, pydar2_specfile s where s.id=v.specfileid group by v.specfileid, s.name;
+create VIEW pydar2_specfile_lastversion AS select max(v.version) as latestversion, v.specfileid, s.filename 
+from pydar2_specfile_version v, pydar2_specfile s where s.id=v.specfileid group by v.specfileid, s.filename;
 
 create view pydar2_specfile_lastversion_and_authority as select distinct
-lv.latestversion, lv.specfileid, lv.name, t.value as authority from
+lv.latestversion, lv.specfileid, lv.filename, t.value as authority from
 pydar2_specfile_lastversion lv, pydar2_specfile_tags t where
 lv.specfileid=t.specfileid and lv.latestversion=t.version and
 lower(t.name) like '# authority';




More information about the commits mailing list