B4J Question huge files quantity

peacemaker · Sep 18, 2023

HI, All

As i can understand, if a system during work generates huge qty of files - it's very bad situation for absolutely any operating system ?
Due to if you need to work with files - you have to list them all first to iterate further, and for huge qty - you'll just hang your app up waiting for listing result...

Correct ?

If yes - maybe someone already tried to solve this situation and made some code functions to store\get files from a huge files storage ?

peacemaker · Sep 19, 2023

I'm trying to make, as usual, universal B4X code (i mean not only tied to Debian or so...)...

Any ideas ?:

'MFS - million files system, for huge file qty
'v.0.1
'(c) https://www.b4x.com/android/forum/members/peacemaker.1637/
Sub Class_Globals
    Private SQL As SQL
    Private OneHourFilesWatcher As FileWatcher
    Private const db_file_name As String = "mfs.db"
    Private mRoot As String
    Private mTimeOut As Long
    Private const MAX_FILES_QTY As Long = 10000000    '10 million
    Public const FILE_NAME_PREFIX As String = "image_"
    Public const FILE_NAME_EXTENSION As String = ".jpg"
    Private LatestFolder As String
    Private timOutdated As Timer
End Sub

'Initializes the object.
'RootFolder - folder where subfolders structure will be created
'StorageTimeOutTicks - milliseconds, after this time interval the oldest files will be deleted
Public Sub Initialize(RootFolder As String, StorageTimeOutTicks As Long) As Boolean
    mTimeOut = StorageTimeOutTicks
    If File.Exists(RootFolder, "") = False Then
        Dim parent As String = File.GetFileParent(RootFolder)
        Dim newfolder As String = File.GetName(RootFolder)
        File.MakeDir(parent, newfolder)
    End If
    mRoot = RootFolder
    Dim free_space As Long = Get_FreeSpace(RootFolder)
    Dim file_size As Long = free_space / MAX_FILES_QTY
    If file_size > 5 Then
        SQL.InitializeSQLite(GetDBFolder("mfs"), db_file_name, True)
        Prepare_DB
        timOutdated.Initialize("timOutdated", 2000)
        timOutdated.Enabled = True
        OneHourFilesWatcher.Initialize("fw")     'Initialize with the event name
        Return True
    Else
        Return False
    End If

End Sub

Private Sub Prepare_DB
    Dim ft As Map
    ft.Initialize
    ft.Put("id", DBUtils.DB_INTEGER)
    ft.Put("name", DBUtils.DB_TEXT)
    ft.Put("size", DBUtils.DB_INTEGER)
    ft.Put("created", DBUtils.DB_INTEGER)    'milliseconds
    ft.Put("lastmodified", DBUtils.DB_INTEGER)    'milliseconds
    ft.Put("path", DBUtils.DB_TEXT)    'full path
    DBUtils.CreateTable(SQL, "fs", ft, "id")
End Sub

'bytes
Private Sub Get_FreeSpace(Fold As String) As Long
    '(ArrayList) [C:\, D:\, E:\, P:\, R:\]
    '(MyMap) {C:\=126100873216, D:\=125829115904, E:\=104857595904, P:\=258932207616, R:\=104857595904}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=96066768896, D:\=91721363456, E:\=83989377024, P:\=242732421120, R:\=83989377024}
    
    If DetectOS = "windows" Then
        Dim drive As String = Fold.SubString2(0, 3).ToUpperCase
    Else If DetectOS = "linux" Then
        Dim drive As String = "/"
    End If
    
    
    Private TotalDrives As List
    TotalDrives.Initialize
    TotalDrives=DiskUtils.GetDrives
    'Log (TotalDrives)
    
    Private TotalDriveCapacity As Map
    TotalDriveCapacity.Initialize
    TotalDriveCapacity=DiskUtils.GetDrivesCapacity
    'Log (TotalDriveCapacity)
    
    Private TotalUsableSpace As Map
    TotalUsableSpace.Initialize
    TotalUsableSpace=DiskUtils.GetUsableSpace
    'Log (TotalUsableSpace)
    
    Private TotalFreeSpace As Map
    TotalFreeSpace.Initialize
    TotalFreeSpace=DiskUtils.GetFreeSpace
    'Log(TotalFreeSpace)
    
    Private Writable As Map
    Writable.Initialize
    Writable=DiskUtils.Writable
    
    Private UsedSpace As Map
    UsedSpace.Initialize
    UsedSpace=DiskUtils.GetUsedSpace
    'Log(UsedSpace)
    

    Dim u As Long = TotalUsableSpace.Get(drive)
    
    Return u
    
    Dim t As Long = TotalDriveCapacity.Get(drive)
    Dim res As Int = (100-((t - u)/t) * 100)
    'Log (res)
    Return res
End Sub

Private Sub DetectOS As String
    Dim os As String = GetSystemProperty("os.name", "").ToLowerCase
    If os.Contains("win") Then
        Return "windows"
    Else If os.Contains("mac") Then
        Return "mac"
    Else
        Return "linux"
    End If
End Sub

Sub Get_Stamp(moment As Long) As String
    Dim b As String, d, E As Long, f As Float
    Dim prevDateF As String = DateTime.DateFormat
    Dim prevTimeF As String = DateTime.TimeFormat
    DateTime.DateFormat = "yyyyMMdd"
    DateTime.TimeFormat = "HHmmss"    ' "HH-mm-ss.SSS(Z)"
    If moment = 0 Then
        d = DateTime.Now
    Else
        d = moment
    End If
    E = d - DateTime.TimeParse(DateTime.Time(d))
    f = E/DateTime.TicksPerSecond
    b = NumberFormat2(f, 0, 3, 3, False)

    Dim stamp As String = DateTime.DAte(d) & "_" & DateTime.Time(d) & b
    
    DateTime.DateFormat = prevDateF
    DateTime.TimeFormat = prevTimeF
    Return stamp
End Sub

#if B4J
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Public Sub GetDBFolder (AppName As String) As String
    Return File.DirData(AppName)
End Sub
#Else
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Public Sub GetDBFolder As String
#If B4A
    Dim rp As RuntimePermissions
    If File.ExternalWritable Then Return rp.GetSafeDirDefaultExternal("") Else Return File.DirInternal
#Else If B4i
    Return File.DirDocuments
#End If
End Sub

#End If

Sub NewFile As String
    Dim Now As Long = DateTime.Now
    Dim fn As String = FILE_NAME_PREFIX & Get_Stamp(Now) & FILE_NAME_EXTENSION
    Dim year As String = NumberFormat2(DateTime.GetYear(Now), 4, 0, 0, False)
    Dim month As String = NumberFormat(DateTime.GetMonth(Now), 2, 0)
    Dim day As String = NumberFormat(DateTime.GetDayOfMonth(Now), 2, 0)
    Dim hour As String = NumberFormat(DateTime.GetHour(Now), 2, 0)
    
    Dim delimiter As String = IIf(DetectOS = "windows", "\", "/")
    Dim folder As String = mRoot & delimiter & year & delimiter & month & delimiter & day & delimiter & hour
    folder = folder.Replace("//", "/").Replace("\\", "\")
    If File.Exists(folder, "") = False Then
        Dim parent As String = File.GetFileParent(folder)
        Dim newfolder As String = File.GetName(folder)
        File.MakeDir(parent, newfolder)
    End If
    
    Dim path As String = File.Combine(folder, fn)
    
    If folder <> LatestFolder Then
        OneHourFilesWatcher.StopWatching
        OneHourFilesWatcher.SetWatchList(Array As String(folder))   'Set the current dir to be watched
        OneHourFilesWatcher.StartWatching
    End If
    
    LatestFolder = folder
    Return path
End Sub

Private Sub fw_CreationDetected(FileName As String)
    Log("fw_CreationDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    Dim Now As Long = DateTime.Now
    Dim L As List:L.Initialize:    Dim ft As Map:ft.Initialize
    
    ft.Put("name", FileName)
    ft.Put("size", File.Size(f, FileName))
    ft.Put("created", Now)    'milliseconds
    ft.Put("lastmodified", Now)    'milliseconds
    ft.Put("path", File.Combine(f, FileName))    'full path
    L.Add(ft)
    DBUtils.InsertMaps(SQL, "fs", L)
End Sub

Private Sub fw_DeletionDetected(FileName As String)
    Log("fw_DeletionDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    Dim where_map As Map:where_map.Initialize
    where_map.Put("path", File.Combine(f, FileName))
    DBUtils.DeleteRecord(SQL, "fs", where_map)
End Sub

Private Sub fw_ModificationDetected(FileName As String)
    Log("fw_ModificationDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    
    Dim where_map As Map:where_map.Initialize
    where_map.Put("path", File.Combine(f, FileName))
    
    Dim ft As Map:ft.Initialize
    ft.Put("size", File.Size(f, FileName))
    ft.Put("lastmodified", File.LastModified(f, FileName))    'milliseconds
    DBUtils.UpdateRecord2(SQL, "fs", ft, where_map)
End Sub

Private Sub timOutdated_Tick
    timOutdated.Enabled = False
    Dim Now As Long = DateTime.Now
    Dim limit As Long = Now - mTimeOut
    Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs WHERE created < '" & limit & "' ORDER by created")

    Dim mapFolders As Map
    mapFolders.Initialize

    Do While r.NextRow
        Dim p As String = r.GetString("path")
        Dim fol As String = File.GetFileParent(p)
        mapFolders.Put(fol, fol)
        'Log(p)
        If File.Exists(p, "") Then
            File.Delete(p, "")
        End If
    Loop
    'Log("all deleted !")
    r.Close
    SQL.ExecNonQuery("DELETE FROM fs WHERE created < '" & limit & "'")
    
    For i = 0 To mapFolders.Size - 1
        Dim fol As String = mapFolders.GetKeyAt(i)
        Dim existing As Int = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs WHERE path LIKE '%" & fol & "%'")
        If existing = 0 Then
            File.Delete(fol, "")    'Delete_Img_Files_Only(fol, "")
        End If
    Next
    timOutdated.Enabled = True
End Sub

Private Sub Delete_Img_Files_Only (folder1 As String, ext As String)
    If folder1 = "" Then Return
    Try
        Dim L As List = File.ListFiles(folder1)
        For i = 0 To L.Size - 1
            Dim fn As String = L.Get(i)
            If File.IsDirectory(folder1, fn) = False Then
                If fn.ToLowerCase.EndsWith(ext.ToLowerCase) Then
                    File.Delete(folder1, fn)
                End If
            End If
        Next
        Log(folder1 & ": deleted all OK")
    Catch
        Log("Delete_All_Files_Only=" & fn & ": " & LastException)
    End Try
End Sub

Magma · Sep 19, 2023

I never used jFileWatcher... and i don't know it's limits - if has any... seems nice...

Also... I didn't understand exactly your scenario... because in some posts you said that you have api receiving the images... and i am staying on that... because if you have an api - i am sure you don't need any library like this (also no dir, ls, commands) you will receive the files with filenames you want and put records at your db only with filenames (and if parsed)...

peacemaker · Sep 19, 2023

Receiving files one by one i have to arrange the storage for long time, it's simple.
Not simple to deal with huge files qty, only this is the problem.

Magma · Sep 19, 2023

peacemaker said:
Receiving files one by one i have to arrange the storage for long time, it's simple.
Not simple to deal with huge files qty, only this is the problem.

but all these files doesn't came automatically ---> using api ---> right ? /// or suddenly open folder and you see 10M (million) files from an empty folder ?

I am saying that from 0 to 10M or from 0 to 1... there is a procedure you have rights to process...

peacemaker · Sep 19, 2023

Not automatically, i'm saving them from HTTP API, assigning my own names. In 24 hours it will be huge list that impossible to list\compare\select\delete.... as the list is loooong and always being updated.

Magma · Sep 19, 2023

peacemaker said:
Not automatically, i'm saving them from HTTP API, assigning my own names. In 24 hours it will be huge list that impossible to list\compare\select\delete.... as the list is loooong and always being updated.

So it is not difficult the same time saving in db a record with filename and if-parsed-field (boolean) ---> not saying to parse the same time...

peacemaker · Sep 19, 2023

Already done something, above. But there are the questions anyway... Will test...

MrKim · Sep 25, 2023

I inherited a customer with an ms access/sql/vba app. The have ONE directory with hundreds of thousands of rich text files. Which are displayed using an old word viewer.
The files are displayed INSTANTLY because the SQL database knows the file name. There is no need to scroll.

Perhaps the solution to your problem is to store the NAMES in an SQL database and use that to retrieve the files.

See @DonManfred post 15.

By the way in windows explorer I have to be VERY careful to NEVER select that directory. If I do the only solution is to open task manager and end the process. I once tried waiting over an hour but it never did finish loading.

MrKim · Sep 25, 2023

MrKim said:
I inherited a customer with an ms access/sql/vba app. The have ONE directory with hundreds of thousands of rich text files. Which are displayed using an old word viewer.
The files are displayed INSTANTLY because the SQL database knows the file name. There is no need to scroll.

Perhaps the solution to your problem is to store the NAMES in an SQL database and use that to retrieve the files.

See @DonManfred post 15.

By the way in windows explorer I have to be VERY careful to NEVER select that directory. If I do the only solution is to open task manager and end the process. I once tried waiting over an hour but it never did finish loading.

The tricky part will be to make sure you NEVER store files without getting them in the DB because there will be no quick way to delete these orphans.

peacemaker · Sep 25, 2023

MrKim said:
store the NAMES in an SQL database and use that to retrieve the files

Sure, already clear and released, need to test

MrKim said:
I once tried waiting over an hour but it never did finish loading

Thanks, you confirmed this my any OS's trouble

Try my class, if interesting

B4X:

'MFS - million files system, for huge file qty
'v.0.12
'(c) https://www.b4x.com/android/forum/members/peacemaker.1637/
Private Sub Class_Globals
    Private SQL As SQL
    Private OneHourFilesWatcher As FileWatcher
    Private const db_file_name As String = "mfs.db"
    Private mRoot As String
    Private mTimeOut As Long
    Private const MAX_FILES_QTY As Long = 10000000    '10 million
    Public const FILE_NAME_PREFIX As String = "image_"
    Public const FILE_NAME_EXTENSION As String = ".jpg"
    Private LatestFolder As String
    Private timOutdated As Timer
End Sub

'Initializes the object.
'RootFolder - folder where subfolders structure will be created
'StorageTimeOutTicks - milliseconds, after this time interval the oldest files will be deleted
Public Sub Initialize(RootFolder As String, StorageTimeOutTicks As Long) As Boolean
    Log("RootFolder = " & RootFolder)
    timOutdated.Initialize("timOutdated", 2000)
    timOutdated.Enabled = False
    mTimeOut = StorageTimeOutTicks
    If File.Exists(RootFolder, "") = False Then
        Dim parent As String = File.GetFileParent(RootFolder)
        Dim newfolder As String = File.GetName(RootFolder)
        File.MakeDir(parent, newfolder)
    End If
    mRoot = RootFolder
    Dim free_space As Long = Get_FreeSpace(RootFolder)
    Dim file_size As Long = free_space / MAX_FILES_QTY
    If file_size > 5 Then
        SQL.InitializeSQLite(GetDBFolder("mfs"), db_file_name, True)
        Prepare_DB
        timOutdated.Enabled = True
        OneHourFilesWatcher.Initialize("fw")     'Initialize with the event name
        Return True
    Else
        Return False
    End If
End Sub

Private Sub SleepMs(ms As Int)
    Sleep(ms)
End Sub

Private Sub Prepare_DB
    Dim ft As Map
    ft.Initialize
    ft.Put("id", DBUtils.DB_INTEGER)
    ft.Put("name", DBUtils.DB_TEXT)
    ft.Put("size", DBUtils.DB_INTEGER)
    ft.Put("created", DBUtils.DB_INTEGER)    'milliseconds
    ft.Put("lastmodified", DBUtils.DB_INTEGER)    'milliseconds
    ft.Put("path", DBUtils.DB_TEXT)    'full path
    DBUtils.CreateTable(SQL, "fs", ft, "id")
End Sub

'bytes
Private Sub Get_FreeSpace(Fold As String) As Long
    '(ArrayList) [C:\, D:\, E:\, P:\, R:\]
    '(MyMap) {C:\=126100873216, D:\=125829115904, E:\=104857595904, P:\=258932207616, R:\=104857595904}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=96066768896, D:\=91721363456, E:\=83989377024, P:\=242732421120, R:\=83989377024}
  
    If DetectOS = "windows" Then
        Dim drive As String = Fold.SubString2(0, 3).ToUpperCase
    Else If DetectOS = "linux" Then
        Dim drive As String = "/"
    End If
  
  
    Private TotalDrives As List
    TotalDrives.Initialize
    TotalDrives=DiskUtils.GetDrives
    'Log (TotalDrives)
  
    Private TotalDriveCapacity As Map
    TotalDriveCapacity.Initialize
    TotalDriveCapacity=DiskUtils.GetDrivesCapacity
    'Log (TotalDriveCapacity)
  
    Private TotalUsableSpace As Map
    TotalUsableSpace.Initialize
    TotalUsableSpace=DiskUtils.GetUsableSpace
    'Log (TotalUsableSpace)
  
    Private TotalFreeSpace As Map
    TotalFreeSpace.Initialize
    TotalFreeSpace=DiskUtils.GetFreeSpace
    'Log(TotalFreeSpace)
  
    Private Writable As Map
    Writable.Initialize
    Writable=DiskUtils.Writable
  
    Private UsedSpace As Map
    UsedSpace.Initialize
    UsedSpace=DiskUtils.GetUsedSpace
    'Log(UsedSpace)
  

    Dim u As Long = TotalUsableSpace.Get(drive)
  
    Return u
  
    Dim t As Long = TotalDriveCapacity.Get(drive)
    Dim res As Int = (100-((t - u)/t) * 100)
    'Log (res)
    Return res
End Sub

Private Sub DetectOS As String
    Dim os As String = GetSystemProperty("os.name", "").ToLowerCase
    If os.Contains("win") Then
        Return "windows"
    Else If os.Contains("mac") Then
        Return "mac"
    Else
        Return "linux"
    End If
End Sub

Private Sub Get_Stamp(moment As Long) As String
    Dim b As String, d, E As Long, f As Float
    Dim prevDateF As String = DateTime.DateFormat
    Dim prevTimeF As String = DateTime.TimeFormat
    DateTime.DateFormat = "yyyyMMdd"
    DateTime.TimeFormat = "HHmmss"    ' "HH-mm-ss.SSS(Z)"
    If moment = 0 Then
        d = DateTime.Now
    Else
        d = moment
    End If
    E = d - DateTime.TimeParse(DateTime.Time(d))
    f = E/DateTime.TicksPerSecond
    b = NumberFormat2(f, 0, 3, 3, False)

    Dim stamp As String = DateTime.DAte(d) & "_" & DateTime.Time(d) & b
  
    DateTime.DateFormat = prevDateF
    DateTime.TimeFormat = prevTimeF
    Return stamp
End Sub

#if B4J
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Private Sub GetDBFolder (AppName As String) As String
    Return File.DirData(AppName)
End Sub
#Else
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Private Sub GetDBFolder As String
#If B4A
    Dim rp As RuntimePermissions
    If File.ExternalWritable Then Return rp.GetSafeDirDefaultExternal("") Else Return File.DirInternal
#Else If B4i
    Return File.DirDocuments
#End If
End Sub

#End If

Sub NewFile As String
    Dim Now As Long = DateTime.Now
    Dim fn As String = FILE_NAME_PREFIX & Get_Stamp(Now) & FILE_NAME_EXTENSION
    Dim year As String = NumberFormat2(DateTime.GetYear(Now), 4, 0, 0, False)
    Dim month As String = NumberFormat(DateTime.GetMonth(Now), 2, 0)
    Dim day As String = NumberFormat(DateTime.GetDayOfMonth(Now), 2, 0)
    Dim hour As String = NumberFormat(DateTime.GetHour(Now), 2, 0)
  
    Dim delimiter As String = IIf(DetectOS = "windows", "\", "/")
    Dim folder As String = mRoot & delimiter & year & delimiter & month & delimiter & day & delimiter & hour
    folder = folder.Replace("//", "/").Replace("\\", "\")
    If File.Exists(folder, "") = False Then
        Dim parent As String = File.GetFileParent(folder)
        Dim newfolder As String = File.GetName(folder)
        File.MakeDir(parent, newfolder)
    End If
  
    Dim path As String = File.Combine(folder, fn)
  
    If folder <> LatestFolder Then
        Log("folder = " & folder)
        OneHourFilesWatcher.SetWatchList(Array As String(folder))   'Set the current dir to be watched
        OneHourFilesWatcher.StartWatching
        SleepMs(0)
        Log("OneHourFilesWatcher started OK")
    End If
  
    LatestFolder = folder
    Return path
End Sub

Private Sub fw_CreationDetected(FileName As String)
    Dim path As String = getPath(FileName)
    'Log("path = " & path)
    If path <> "" Then    'already exists
        fw_ModificationDetected(FileName)
        Return
    End If
  
'    Log("fw_CreationDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    'Log("CreationDetected = " & DateTime.Now)
    Dim Now As Long = File.LastModified(f, FileName)    'DateTime.Now
    'Log("LastModified = " & Now)
  
    Dim L As List:L.Initialize:    Dim ft As Map:ft.Initialize 
    ft.Put("name", FileName)
    ft.Put("size", File.Size(f, FileName))
    ft.Put("created", Now)    'milliseconds
    ft.Put("lastmodified", Now)    'milliseconds
    ft.Put("path", File.Combine(f, FileName))    'full path
    L.Add(ft)
    DBUtils.InsertMaps(SQL, "fs", L)
End Sub

Private Sub fw_DeletionDetected(FileName As String)
    Log("fw_DeletionDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    Dim where_map As Map:where_map.Initialize
    where_map.Put("path", File.Combine(f, FileName))
    DBUtils.DeleteRecord(SQL, "fs", where_map)
End Sub

Private Sub fw_ModificationDetected(FileName As String)
    'Log("ModificationDetected = " & DateTime.Now)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    Dim beforemodified As Long = getLastmodified(FileName) 
    Dim LastModified As Long = File.LastModified(f, FileName)
    Dim beforesize As Long = getSize(FileName)
    Dim size As Long = File.Size(f, FileName)
  
    If LastModified <> beforemodified Or beforesize <> size Then
        Log("fw_ModificationDetected = " & FileName)
        Dim where_map As Map:where_map.Initialize
        where_map.Put("path", File.Combine(f, FileName))
      
        Dim ft As Map:ft.Initialize
        ft.Put("size", File.Size(f, FileName))
        ft.Put("lastmodified", File.LastModified(f, FileName))    'milliseconds
        DBUtils.UpdateRecord2(SQL, "fs", ft, where_map)
    End If
End Sub

Private Sub fw_Overflow
    Log("fw_Overflow")
End Sub

Private Sub timOutdated_Tick
    timOutdated.Enabled = False
    Dim Now As Long = DateTime.Now
    Dim limit As Long = Now - mTimeOut
    Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs WHERE created < '" & limit & "' ORDER by created")

    Dim mapFolders As Map
    mapFolders.Initialize

    Do While r.NextRow
        Dim p As String = r.GetString("path")
        Dim fol As String = File.GetFileParent(p)
        mapFolders.Put(fol, fol)
        'Log(p)
        If File.Exists(p, "") Then
            File.Delete(p, "")
        End If
    Loop
    'Log("all deleted !")
    r.Close
    SQL.ExecNonQuery("DELETE FROM fs WHERE created < '" & limit & "'")
  
    For i = 0 To mapFolders.Size - 1
        Dim fol As String = mapFolders.GetKeyAt(i)
        Dim existing As Int = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs WHERE path LIKE '%" & fol & "%'")
        If existing = 0 Then
            File.Delete(fol, "")    'Delete_Img_Files_Only(fol, "")
        End If
    Next
    timOutdated.Enabled = True
End Sub

Private Sub Delete_Img_Files_Only (folder1 As String, ext As String)
    If folder1 = "" Then Return
    Try
        Dim L As List = File.ListFiles(folder1)
        For i = 0 To L.Size - 1
            Dim fn As String = L.Get(i)
            If File.IsDirectory(folder1, fn) = False Then
                If fn.ToLowerCase.EndsWith(ext.ToLowerCase) Then
                    File.Delete(folder1, fn)
                End If
            End If
        Next
        Log(folder1 & ": deleted all OK")
    Catch
        Log("Delete_All_Files_Only=" & fn & ": " & LastException)
    End Try
End Sub

Private Sub DeleteFolder (folder As String, killself As Boolean)
    For Each f As String In File.ListFiles(folder)
        If File.IsDirectory(folder, f) Then
            DeleteFolder(File.Combine(folder, f), True)
        End If
        File.Delete(folder, f)
    Next
    If killself Then
        File.Delete(folder, "")
    End If
End Sub

Sub Clear_All
    OneHourFilesWatcher.StopWatching
    Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs ORDER BY created DESC")

    Do While r.NextRow
        Dim p As String = r.GetString("path")
        Dim fol As String = File.GetFileParent(p)
        If File.Exists(fol, "") Then
            DeleteFolder(fol, True)
        End If
    Loop
    r.Close
    DeleteFolder(mRoot, False)
    DBUtils.ClearTable(SQL, "fs")
    Log("All cleared !")
End Sub


Public Sub getPath(filename As String) As String
    Dim path As String = SQL.ExecQuerySingleResult("SELECT path FROM fs WHERE path LIKE '%" & filename & "%'")
    If path = Null Then
        path = ""
    End If
    Return path
End Sub

Public Sub getCreated(filename As String) As Long
    Dim created As Long
    Dim path As String = getPath(filename)
    If path <> "" Then
        created = SQL.ExecQuerySingleResult2("SELECT created FROM fs WHERE path = ?", Array As String(path))
    End If
    Return created
End Sub

Public Sub getLastmodified(filename As String) As Long
    Dim Lastmodified As Long
    Dim path As String = getPath(filename)
    If path <> "" Then
        Lastmodified = SQL.ExecQuerySingleResult2("SELECT lastmodified FROM fs WHERE path = ?", Array As String(path))
    End If
    Return Lastmodified
End Sub

Public Sub getSize(filename As String) As Long
    Dim Size As Long
    Dim path As String = getPath(filename)
    If path <> "" Then
        Size = SQL.ExecQuerySingleResult2("SELECT size FROM fs WHERE path = ?", Array As String(path))
    End If
    Return Size
End Sub

Public Sub getLatestFiles(qty As Int) As List
    Dim L As List = DBUtils.ExecuteList(SQL, "SELECT path FROM fs ORDER BY lastmodified ASC", Null, qty)
    Return L
End Sub

Public Sub getTotalQty As Long
    Dim a As Long = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs")
    Return a
End Sub

Public Sub Delete(fullpath As String) As Boolean
    Dim a As Boolean = File.Delete("", fullpath)
    If a Then
        SQL.ExecNonQuery("DELETE FROM fs WHERE path = '" & fullpath & "'")
    End If
    Return a
End Sub

Public Sub Delete2(filename As String) As Boolean
    Dim path As String = getPath(filename)
    Return Delete(path)
End Sub

peacemaker · Oct 2, 2023

Actually, the latest class version is here, let it be useful for all:

B4X:

'MFS - million files system, for huge file qty
'v.0.13
'(c) https://www.b4x.com/android/forum/members/peacemaker.1637/
Private Sub Class_Globals
    Private SQL As SQL
    Private OneHourFilesWatcher As FileWatcher
    Private db_file_name As String = "mfs.db"
    Private mRoot As String
    Private mTimeOut As Int
    Private const MAX_FILES_QTY As Long = 10000000    '10 million
    Private FreePercentLimit As Int = 20
    Public const FILE_NAME_PREFIX As String = "image_"
    Public const FILE_NAME_EXTENSION As String = ".jpg"
    Private LatestFolder As String
    Private timOutdated As Timer
End Sub

'Initializes the object.
'RootFolder - folder where subfolders structure will be created
'StorageTimeOutTicks - milliseconds, after this time interval the oldest files will be deleted
Public Sub Initialize(RootFolder As String, dbfn As String, StorageTimeOutTicks As Long, StorageFreePercentLimit As Int) As Boolean
    Log("RootFolder = " & RootFolder)
    If dbfn = "" Then
        db_file_name = "mfs.db"
    Else
        db_file_name = dbfn
    End If
    If StorageFreePercentLimit <= 0 Then
        FreePercentLimit = 20
    Else
        FreePercentLimit = StorageFreePercentLimit
    End If
    timOutdated.Initialize("timOutdated", 2000)
    timOutdated.Enabled = False
    mTimeOut = StorageTimeOutTicks
    If File.Exists(RootFolder, "") = False Then
        Dim parent As String = File.GetFileParent(RootFolder)
        Dim newfolder As String = File.GetName(RootFolder)
        File.MakeDir(parent, newfolder)
    End If
    mRoot = RootFolder
    Dim free_space As Long = Get_FreeSpace(RootFolder)
    Dim file_size As Long = free_space / MAX_FILES_QTY
    If file_size > 5 Then
        SQL.InitializeSQLite(GetDBFolder("mfs"), db_file_name, True)
        Prepare_DB
        timOutdated.Enabled = True
        OneHourFilesWatcher.Initialize("fw")     'Initialize with the event name
        Return True
    Else
        Return False
    End If
End Sub

Private Sub SleepMs(ms As Int)
    Sleep(ms)
End Sub

Private Sub Prepare_DB
    Dim ft As Map
    ft.Initialize
    ft.Put("id", DBUtils.DB_INTEGER)
    ft.Put("name", DBUtils.DB_TEXT)
    ft.Put("size", DBUtils.DB_INTEGER)
    ft.Put("created", DBUtils.DB_INTEGER)    'milliseconds
    ft.Put("lastmodified", DBUtils.DB_INTEGER)    'milliseconds
    ft.Put("path", DBUtils.DB_TEXT)    'full path
    DBUtils.CreateTable(SQL, "fs", ft, "id")
End Sub

'bytes
Private Sub Get_FreeSpace(Fold As String) As Long
    '(ArrayList) [C:\, D:\, E:\, P:\, R:\]
    '(MyMap) {C:\=126100873216, D:\=125829115904, E:\=104857595904, P:\=258932207616, R:\=104857595904}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=96066768896, D:\=91721363456, E:\=83989377024, P:\=242732421120, R:\=83989377024}
    
    If DetectOS = "windows" Then
        Dim drive As String = Fold.SubString2(0, 3).ToUpperCase
    Else If DetectOS = "linux" Then
        Dim drive As String = "/"
    End If
    
    
    Private TotalDrives As List
    TotalDrives.Initialize
    TotalDrives=DiskUtils.GetDrives
    'Log (TotalDrives)
    
    Private TotalDriveCapacity As Map
    TotalDriveCapacity.Initialize
    TotalDriveCapacity=DiskUtils.GetDrivesCapacity
    'Log (TotalDriveCapacity)
    
    Private TotalUsableSpace As Map
    TotalUsableSpace.Initialize
    TotalUsableSpace=DiskUtils.GetUsableSpace
    'Log (TotalUsableSpace)
    
    Private TotalFreeSpace As Map
    TotalFreeSpace.Initialize
    TotalFreeSpace=DiskUtils.GetFreeSpace
    'Log(TotalFreeSpace)
    
    Private Writable As Map
    Writable.Initialize
    Writable=DiskUtils.Writable
    
    Private UsedSpace As Map
    UsedSpace.Initialize
    UsedSpace=DiskUtils.GetUsedSpace
    'Log(UsedSpace)
    

    Dim u As Long = TotalUsableSpace.Get(drive)
    
    Return u
End Sub

'%
Private Sub Get_FreeSpacePercent(Fold As String) As Long
    '(ArrayList) [C:\, D:\, E:\, P:\, R:\]
    '(MyMap) {C:\=126100873216, D:\=125829115904, E:\=104857595904, P:\=258932207616, R:\=104857595904}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=30034104320, D:\=34107752448, E:\=20868218880, P:\=16199786496, R:\=20868218880}
    '(MyMap) {C:\=96066768896, D:\=91721363456, E:\=83989377024, P:\=242732421120, R:\=83989377024}
    
    If DetectOS = "windows" Then
        Dim drive As String = Fold.SubString2(0, 3).ToUpperCase
    Else If DetectOS = "linux" Then
        Dim drive As String = "/"
    End If
    
    
    Private TotalDrives As List
    TotalDrives.Initialize
    TotalDrives=DiskUtils.GetDrives
    'Log (TotalDrives)
    
    Private TotalDriveCapacity As Map
    TotalDriveCapacity.Initialize
    TotalDriveCapacity=DiskUtils.GetDrivesCapacity
    'Log (TotalDriveCapacity)
    
    Private TotalUsableSpace As Map
    TotalUsableSpace.Initialize
    TotalUsableSpace=DiskUtils.GetUsableSpace
    'Log (TotalUsableSpace)
    
    Private TotalFreeSpace As Map
    TotalFreeSpace.Initialize
    TotalFreeSpace=DiskUtils.GetFreeSpace
    'Log(TotalFreeSpace)
    
    Private Writable As Map
    Writable.Initialize
    Writable=DiskUtils.Writable
    
    Private UsedSpace As Map
    UsedSpace.Initialize
    UsedSpace=DiskUtils.GetUsedSpace
    'Log(UsedSpace)
    

    Dim u As Long = TotalUsableSpace.Get(drive)
    Dim t As Long = TotalDriveCapacity.Get(drive)
    Dim res As Int = (100-((t - u)/t) * 100)
    'Log (res)
    Return res
End Sub

Private Sub DetectOS As String
    Dim os As String = GetSystemProperty("os.name", "").ToLowerCase
    If os.Contains("win") Then
        Return "windows"
    Else If os.Contains("mac") Then
        Return "mac"
    Else
        Return "linux"
    End If
End Sub

Private Sub Get_Stamp(moment As Long) As String
    Dim b As String, d, E As Long, f As Float
    Dim prevDateF As String = DateTime.DateFormat
    Dim prevTimeF As String = DateTime.TimeFormat
    DateTime.DateFormat = "yyyyMMdd"
    DateTime.TimeFormat = "HHmmss"    ' "HH-mm-ss.SSS(Z)"
    If moment = 0 Then
        d = DateTime.Now
    Else
        d = moment
    End If
    E = d - DateTime.TimeParse(DateTime.Time(d))
    f = E/DateTime.TicksPerSecond
    b = NumberFormat2(f, 0, 3, 3, False)

    Dim stamp As String = DateTime.DAte(d) & "_" & DateTime.Time(d) & b
    
    DateTime.DateFormat = prevDateF
    DateTime.TimeFormat = prevTimeF
    Return stamp
End Sub

#if B4J
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Private Sub GetDBFolder (AppName As String) As String
    Return File.DirData(AppName)
End Sub
#Else
'Returns the path to a folder where you can create a database, preferably on the secondary storage.
Private Sub GetDBFolder As String
#If B4A
    Dim rp As RuntimePermissions
    If File.ExternalWritable Then Return rp.GetSafeDirDefaultExternal("") Else Return File.DirInternal
#Else If B4i
    Return File.DirDocuments
#End If
End Sub

#End If

Sub NewFile As String
    Dim Now As Long = DateTime.Now
    Dim fn As String = FILE_NAME_PREFIX & Get_Stamp(Now) & FILE_NAME_EXTENSION
    Dim year As String = NumberFormat2(DateTime.GetYear(Now), 4, 0, 0, False)
    Dim month As String = NumberFormat(DateTime.GetMonth(Now), 2, 0)
    Dim day As String = NumberFormat(DateTime.GetDayOfMonth(Now), 2, 0)
    Dim hour As String = NumberFormat(DateTime.GetHour(Now), 2, 0)
    
    Dim delimiter As String = IIf(DetectOS = "windows", "\", "/")
    Dim folder As String = mRoot & delimiter & year & delimiter & month & delimiter & day & delimiter & hour
    folder = folder.Replace("//", "/").Replace("\\", "\")
    If File.Exists(folder, "") = False Then
        Dim parent As String = File.GetFileParent(folder)
        Dim newfolder As String = File.GetName(folder)
        File.MakeDir(parent, newfolder)
    End If
    
    Dim path As String = File.Combine(folder, fn)
    
    If folder <> LatestFolder Then
'        Log("folder = " & folder)
        OneHourFilesWatcher.StopWatching
        OneHourFilesWatcher.SetWatchList(Array As String(folder))   'Set the current dir to be watched
        OneHourFilesWatcher.StartWatching
        SleepMs(0)
'        Log("OneHourFilesWatcher started OK")
    End If
    
    LatestFolder = folder
    'fw_CreationDetected(fn)
    Return path
End Sub

Private Sub fw_CreationDetected(FileName As String)
    Dim path As String = getPath(FileName)
    'Log("path = " & path)
    If path <> "" Then    'already exists
        fw_ModificationDetected(FileName)
        Return
    End If
    
'    Log("fw_CreationDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    'Log("CreationDetected = " & DateTime.Now)
    Dim Now As Long = File.LastModified(f, FileName)    'DateTime.Now
    'Log("LastModified = " & Now)
    
    Dim L As List:L.Initialize:    Dim ft As Map:ft.Initialize   
    ft.Put("name", FileName)
    ft.Put("size", File.Size(f, FileName))
    ft.Put("created", Now)    'milliseconds
    ft.Put("lastmodified", Now)    'milliseconds
    ft.Put("path", File.Combine(f, FileName))    'full path
    L.Add(ft)
    DBUtils.InsertMaps(SQL, "fs", L)
End Sub

Private Sub fw_DeletionDetected(FileName As String)
    Log("fw_DeletionDetected = " & FileName)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    Dim where_map As Map:where_map.Initialize
    where_map.Put("path", File.Combine(f, FileName))
    DBUtils.DeleteRecord(SQL, "fs", where_map)
End Sub

Private Sub fw_ModificationDetected(FileName As String)
    'Log("ModificationDetected = " & DateTime.Now)
    Dim f As String = OneHourFilesWatcher.GetWatchList.Get(0)
    Dim beforemodified As Long = getLastmodified(FileName)   
    Dim LastModified As Long = File.LastModified(f, FileName)
    Dim beforesize As Long = getSize(FileName)
    Dim size As Long = File.Size(f, FileName)
    
    If LastModified <> beforemodified Or beforesize <> size Then
        Log("fw_ModificationDetected = " & FileName)
        Dim where_map As Map:where_map.Initialize
        where_map.Put("path", File.Combine(f, FileName))
        
        Dim ft As Map:ft.Initialize
        ft.Put("size", File.Size(f, FileName))
        ft.Put("lastmodified", File.LastModified(f, FileName))    'milliseconds
        DBUtils.UpdateRecord2(SQL, "fs", ft, where_map)
    End If
End Sub

Private Sub fw_Overflow
    Log("fw_Overflow")
End Sub

Private Sub timOutdated_Tick
    timOutdated.Enabled = False
    Dim Now As Long = DateTime.Now
    Dim limit As Long = Now - Abs(mTimeOut)
    'Log("limit = " & limit)
    Dim lim As Int = Get_FreeSpacePercent(mRoot)   
    If lim < FreePercentLimit Then
        Dim L As List = getOldestFiles(1)
        Dim t As Long = getLastmodified(L.Get(0))
        limit = t + DateTime.TicksPerHour
    End If
    Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs WHERE created < '" & limit & "' ORDER by created")

    Dim mapFolders As Map
    mapFolders.Initialize

    Do While r.NextRow
        Dim p As String = r.GetString("path")
        'Dim created As Long = r.GetLong("created")
        'Log("created = " & created)
        Dim fol As String = File.GetFileParent(p)
        mapFolders.Put(fol, fol)
        'Log(p)
        If File.Exists(p, "") Then
            Delete(p)
        End If
    Loop
    'Log("all deleted !")
    r.Close
    
    For i = 0 To mapFolders.Size - 1
        Dim fol As String = mapFolders.GetKeyAt(i)
        Dim existing As Int = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs WHERE path LIKE '%" & fol & "%'")
        If existing = 0 Then
            File.Delete(fol, "")
        End If
    Next
    timOutdated.Enabled = True
End Sub

Private Sub Delete_Img_Files_Only (folder1 As String, ext As String)
    If folder1 = "" Then Return
    Try
        Dim L As List = File.ListFiles(folder1)
        For i = 0 To L.Size - 1
            Dim fn As String = L.Get(i)
            If File.IsDirectory(folder1, fn) = False Then
                If fn.ToLowerCase.EndsWith(ext.ToLowerCase) Then
                    File.Delete(folder1, fn)
                End If
            End If
        Next
        Log(folder1 & ": deleted all OK")
    Catch
        Log("Delete_All_Files_Only=" & fn & ": " & LastException)
    End Try
End Sub

Private Sub DeleteFolder (folder As String, killself As Boolean)
    For Each f As String In File.ListFiles(folder)
        If File.IsDirectory(folder, f) Then
            DeleteFolder(File.Combine(folder, f), True)
        End If
        File.Delete(folder, f)
    Next
    If killself Then
        File.Delete(folder, "")
    End If
End Sub

Sub Clear_All
    OneHourFilesWatcher.StopWatching
    Dim r As ResultSet = SQL.ExecQuery("SELECT path FROM fs ORDER BY created DESC")

    Do While r.NextRow
        Dim p As String = r.GetString("path")
        Dim fol As String = File.GetFileParent(p)
        If File.Exists(fol, "") Then
            DeleteFolder(fol, True)
        End If
    Loop
    r.Close
    DeleteFolder(mRoot, False)
    DBUtils.ClearTable(SQL, "fs")
    Log("All cleared !")
End Sub


Public Sub getPath(filename As String) As String
    Try
        Dim path As String = SQL.ExecQuerySingleResult("SELECT path FROM fs WHERE path LIKE '%" & filename & "%'")
    Catch
        Dim path As String = Null
    End Try
    If path = Null Then
        path = ""
    End If
    Return path
End Sub

Public Sub getCreated(filename As String) As Long
    Dim created As Long
    Dim path As String = getPath(filename)
    If path <> "" Then
        created = SQL.ExecQuerySingleResult2("SELECT created FROM fs WHERE path = ?", Array As String(path))
    End If
    Return created
End Sub

Public Sub getLastmodified(filename As String) As Long
    Dim Lastmodified As Long
    Dim path As String = getPath(filename)
    If path <> "" Then
        Try
            Lastmodified = SQL.ExecQuerySingleResult2("SELECT lastmodified FROM fs WHERE path = ?", Array As String(path))
        Catch
            Lastmodified = -1
        End Try
    End If
    Return Lastmodified
End Sub

Public Sub getSize(filename As String) As Long
    Dim Size As Long
    Dim path As String = getPath(filename)
    If path <> "" Then
        Size = SQL.ExecQuerySingleResult2("SELECT size FROM fs WHERE path = ?", Array As String(path))
    End If
    Return Size
End Sub

Public Sub getLatestFiles(qty As Int) As List
    Dim L As List
    Try
        L = DBUtils.ExecuteList(SQL, "SELECT path FROM fs ORDER BY lastmodified DESC", Null, qty)
    Catch
        Log("getLatestFiles.error = " & LastException.Message)
    End Try
    Return L
End Sub

Public Sub getOldestFiles(qty As Int) As List
    Dim L As List
    Try
        L = DBUtils.ExecuteList(SQL, "SELECT path FROM fs ORDER BY lastmodified ASC", Null, qty)
    Catch
        Log("getOldestFiles.error = " & LastException.Message)
    End Try
    Return L
End Sub

Public Sub getTotalQty As Long
    Dim a As Long = SQL.ExecQuerySingleResult("SELECT count(*) FROM fs")
    Return a
End Sub

Public Sub Delete(fullpath As String) As Boolean
    Dim a As Boolean = File.Delete("", fullpath)
    If a Then
        SQL.ExecNonQuery("DELETE FROM fs WHERE path = '" & fullpath & "'")
    End If
    Return a
End Sub

Public Sub Delete2(filename As String) As Boolean
    Dim path As String = getPath(filename)
    Return Delete(path)
End Sub

B4J Question huge files quantity

peacemaker

Expert

peacemaker

Expert

Attachments

Magma

Expert

peacemaker

Expert

Magma

Expert

peacemaker

Expert

Magma

Expert

peacemaker

Expert

MrKim

Well-Known Member

MrKim

Well-Known Member

peacemaker

Expert

Attachments

peacemaker

Expert

Similar Threads

B4J Question huge files quantity

Expert

Expert

Attachments

Expert

Expert

Expert

Expert

Expert

Expert

Well-Known Member

Well-Known Member

Expert

Attachments

Expert

Similar Threads

Privacy & Transparency

Privacy & Transparency