Of course, thanks for your opinion!
I also think it is due to disk.
Regards
Qatan escribió:
> Hello Rafa,
>
> I am not sure, may be wrong but I would say also that the fisical
> access of the HD Head divided in different threads would increase the
> time to access the same ammount of data compared to only one thread.
> This is due to the fisical comings and goings of the Head over the
> disk surface.
> I don't know for SDD but maybe it would have a similar effect
> because multi threading would not be an advantage for disk access,
> only for CPU.
>
> Qatan
>
>
> ----- Original Message ----- From: "rafa" <r...@tesipro.com>
> To: "Harbour Project Main Developer List." <harbour@harbour-project.org>
> Sent: Monday, 10 de May de 2010 06:20
> Subject: [Harbour] Slow create index under threads
>
>
>> Hello,
>> I'm trying to do a re indexing using threads, but according to tests
>> under Windows Vista,
>> Quarda Intel Core, 4 Gig RAM, the tests are not entirely satisfactory;
>>
>> The basic idea was to determine the number of threads that want to
>> implement,
>> N_THREADS, which is passed by parameter, for testing purposes.
>>
>> Well, times;
>> 1 thread = 29 minutes THE WINNER!
>> 4 threads = 93 minutes WoW!
>> 7 threads = 68 minutes
>> 5 Threads = 64 minutes
>>
>> Something to do this by penalizing Harbour indexing, or the code above,
>> you have
>> thing excessively penalized.
>>
>> ---------------- SPANISH -------------------------------------------
>> Hola,
>>
>> Estoy intentando hacer una re indexación usando hilos, pero según
>> pruebas bajo Windows Vista,
>> Intel Quard Core, 4 Gigas RAM, los test no son del todo satisfactorios;
>>
>> La idea básica es determinar la cantidad de hilos que queremos poner en
>> marcha,
>> N_THREADS, que es pasado por parámetro, para poder realizar pruebas.
>>
>> Pues bien, los tiempos ;
>> 1 Hilo = 29 minutos THE WINNER!
>> 4 Hilos = 93 minutos WoW!!
>> 7 hilos = 68 minutos
>> 5 Hilos = 64 minutos
>>
>> Algo hacer Harbour que esta penalizando la indexación, o el código
>> expuesto, tiene alguna
>> cosa que penaliza en exceso.
>>
>> Cualquier comentario es bienvenido para mejorar este aspecto.
>>
>> //---------------------------------------------- source code
>> ----------------------------------------------------------
>> /*
>> Example multiThreads index.
>> One thread by table , and one thread by index.
>> 2010 Rafa Carmona
>>
>> Thread Main
>> |---------> table for test.dbf
>> | |----> Thread child index fname
>> | |
>> | |----->Thread child index fcode
>>
>> c:\> ..\..\bin\win\bcc\hbmk2 -mt indexthread -lhbcpage
>> -Le:\harbour\trunk\harbour\lib\win\bcc
>>
>> New code, now, if thread dead, new thread create!
>>
>> */
>> #include "hbclass.ch"
>> #include "hbthread.ch"
>> #include "common.ch"
>> #include "inkey.ch"
>> #include "FileIO.ch"
>>
>> REQUEST HB_CODEPAGE_ES850, HB_CODEPAGE_ES850C
>> REQUEST HB_LANG_ES
>>
>> static N_THREADS := 5
>>
>> STATIC nTecla
>> STATIC s_num_procesos
>> STATIC s_aLineas := { }
>> static s_hMutex
>> static s_hHandle_File
>> static s_nCount_Errores := 0
>>
>> proc Main( nHilos )
>> Local nSeconds
>> Local cDbf, lProcesa := .F.
>> Local aFicheros, x
>> Local aDbfs, aNtxs, aKeys , aDesc, aFor, aSel
>> Local aThreads := {}
>> Local nProceso := 0, nLinea, lSalir := .F., nIndex
>> Local nLen_Table
>> Local nPosTable, cCadena, cLine, aTokens, i
>> Local nPos_Column := 1, g
>>
>>
>> DEFAULT nHilos TO 0
>>
>> if !empty( nHilos )
>> N_THREADS := val( nHilos )
>> endif
>>
>> //HB_SetCodePage( "ES850" ) En xHarbour
>> set( _SET_CODEPAGE, "ES850C" )
>>
>> HB_LANGSELECT('ES')
>> Set( _SET_LANGUAGE, "ES" )
>>
>>
>> setmode( 25,130 )
>> cls
>>
>> @01,0 SAY padc( hb_ansitooem( " Indexación multihilo. Rafa Carmona"
>> ), 80 )COLOR "N*/W*"
>> @23,0 SAY padc( "Pulse ESC para cancelar." , 80) COLOR "R+/N"
>>
>> // ONLY for TEST, if necesary many many DBF , with millions records
>> for test.
>> // My test is over 200 dbfs, with 12GB total size, without NTX.
>> aDbfs := { "test", "test2" } // Arrays files dbf
>>
>> aNtxs := { { "fname", "fcode" },; // files index for test
>> { "fName2" } } // files index for test2
>>
>> aKeys := { { "name", "code" },;
>> { "dtos(fecha)+str(code)" } } // Expresions
>>
>> aFor := { { "", ""}, {""} }
>>
>>
>> nLen_Table := len( aDbfs )
>> nPosTable := 1
>> nSeconds := Seconds()
>> s_num_procesos := 0
>> s_hMutex := hb_mutexCreate()
>> s_hHandle_File := FCreate( "indexpms.log" )
>>
>> for g := 1 to N_THREADS // Posicion en la columa por cada numero de
>> hilo
>> aadd( s_aLineas, nPos_Column )
>> nPos_Column += 16
>> next
>>
>>
>> while nPosTable <= nLen_Table
>>
>> if ( nTecla := inkey() ) = K_ESC
>> exit
>> endif
>>
>> if N_THREADS = s_num_procesos // No se ha muerto ningun proceso
>> loop
>> endif
>>
>> cDbf := aDbfs[ nPosTable ]
>>
>>
>> if file( cDbf+".dbf" )
>> hb_mutexLock( s_hMutex )
>> s_num_procesos++
>> hb_mutexUnLock( s_hMutex )
>> hb_threadStart( @aCreateIndexe(), cDbf, aNtxs[ nPosTable ],
>> aKeys[ nPosTable ], aFor[ nPosTable ] )
>> endif
>>
>> nPosTable++
>> end while
>>
>> @23,1 SAY "Espere, terminado reindexaciones pendientes..." + space( 50
>> ) COLOR "R*/N"
>> hb_threadWaitForAll() // Esperamos a los ultimos.
>> @23,1 SAY "Proceso de indexacion terminado..." + Str( (Seconds() -
>> nSeconds) /60 ) + space( 50 ) COLOR "W+/N"
>>
>> FClose( s_hHandle_File )
>>
>> return
>>
>>
>> function aCreateIndexe( cFile, aNtx, aExpr, aFor )
>> Local nContador := 1
>> Local cFileNtx, cExpr
>> Local aThreads := {}
>> Local cAlias, cFor
>> Local nP, x, oError, lreturn := .f.
>>
>> use ( cFile )
>> if neterr()
>> hb_mutexLock( s_hMutex )
>> s_num_procesos--
>> s_nCount_Errores++
>> hb_mutexUnLock( s_hMutex )
>> @22,1 SAY "[ "+ alltrim( str( s_nCount_Errores ))+" ] Error de
>> apertura en el fichero : "+ cFile + space( 10 )
>> fwrite( s_hHandle_File, "Error de apertura en el fichero : "+
>> cFile + Hb_OsNewline() )
>> return nil
>> endif
>>
>> cAlias := alias()
>> hb_dbDetach( cAlias ) // Libero el alias
>>
>> for x := 1 to len( s_aLineas )
>> nPosLinea := s_aLineas[ x ]
>> if !empty( nPosLinea )
>> s_aLineas[x] := 0 // Quita del array esa linea disponible
>> para pintar
>> exit
>> endif
>> next
>>
>> @ 2, nPosLinea CLEAR TO 15, nPosLinea + 14 // Limpiamos cuadrado
>> hb_dispOutAt( 2, nPosLinea , cFile )
>>
>> for each cFileNtx in aNtx
>> cExpr := aExpr[ cFileNtx:__enumindex ]
>> cFor := aFor[ cFileNtx:__enumindex ]
>> nPos := cFileNtx:__enumindex
>> delete file ( cFileNtx +".ntx") // Se borra el fichero, por
>> ahorro de espacio en disco.
>> aadd( aThreads, hb_threadStart( @crea(), cAlias,cExpr,
>> cFileNtx, cFor, nPos, nPosLinea ) )
>> if ( nTecla := inkey() ) = K_ESC // No protegemos variable
>> static
>> exit
>> endif
>> next
>>
>> aEval( aThreads, { |x| hb_threadJoin( x ) } ) // Espera que
>> termine los hilos hijos
>>
>> hb_dbRequest( cAlias, , , .T.) // Restaura el alias
>> close
>>
>> // Vuelve a colocar la linea como disponible para pintar
>> s_aLineas[x] := nPosLinea
>> hb_mutexLock( s_hMutex )
>> s_num_procesos--
>> hb_mutexUnLock( s_hMutex )
>>
>> RETURN .T.
>>
>> proc crea( cAlias, cExpr, cFileNtx, cFor, nPos, nPosLinea )
>> Local nContador := 1
>>
>> hb_dbRequest( cAlias, , , .T.) // Restaura el alias
>> if empty( cfor )
>> INDEX ON &(cExpr) TO &(cFileNtx) ;
>> EVAL {|| hb_dispOutAt( nPos +3, nPosLinea, padr(
>> cFileNtx,8 )+ "-"+alltrim( hb_valtostr( nContador) ), "GR+/N" ),
>> nContador += INT( LASTREC() / 100 ) , .T. } ;
>> EVERY INT( LASTREC() / 100 )
>> // EVAL {|| hb_dispOutAt( nPos +2, nPosLinea, padr(
>> cFileNtx,8 )+ "-"+ padr( cExpr,20)+ " # " +alltrim( hb_valtostr(
>> nContador) ), "GR+/N" ), nContador += INT( LASTREC() / 100 ) , .T. } ;
>> else
>> INDEX ON &(cExpr) TO &(cFileNtx) FOR &(cFor) ;
>> EVAL {|| hb_dispOutAt( nPos +3, nPosLinea, padr(
>> cFileNtx,8 )+ "-"+alltrim( hb_valtostr( nContador) ), "GR+/N" ),
>> nContador += INT( LASTREC() / 100 ) , .T. } ;
>> EVERY INT( LASTREC() / 100 )
>> endif
>> hb_dbDetach( cAlias ) // Libera el alias
>>
>>
>> return
>> _______________________________________________
>> Harbour mailing list (attachment size limit: 40KB)
>> Harbour@harbour-project.org
>> http://lists.harbour-project.org/mailman/listinfo/harbour
>
> _______________________________________________
> Harbour mailing list (attachment size limit: 40KB)
> Harbour@harbour-project.org
> http://lists.harbour-project.org/mailman/listinfo/harbour
>
_______________________________________________
Harbour mailing list (attachment size limit: 40KB)
Harbour@harbour-project.org
http://lists.harbour-project.org/mailman/listinfo/harbour
