DAMemDataTable - Memory-leaks with StoreStringAsReference option active

Hello,

I have discovered a bug in the way large string fields are handled. If I utilize an index on such field, every search leaks memory. I have analysed the cause and I think this is caused by the raw manipulations of FKeyBuffer’s memory. I have prepared a small sample project to demostrate the issue. When you click on the button, the data table is populated and several indexed searches are performed (each search leaking memory of the searched string).

Regard from Prague,
Jaroslav

daSample.zip (52.2 KB)

Thanks, logged as bugs://78011

bugs://78011 got closed with status fixed.

good catch.
fix will be in this week beta

Hello,
unfortunately the fix has severe adverse effects. In some circumstances it tries to free already freed memory (and not shortly after). Causing random objects to be deallocated or random AV to be triggered.

The original sample project does not trigger this behavior, but if the sequence of SetKey, Post, GotoKey is repeatedly used then the key buffer kiSave will get a hard copy of the kiLookup buffer before the last use. Triggering a free when the DataTable is closed.

Regard from Prague,
Jaroslav

can you create a simple testcase that reproduced this issue, pls?

…see attached sample.

Regards from Prague,
Jaroslav

daSample2.zip (52.7 KB)

pls update uDAMemDataset as

procedure TDAMemoryDataset.FreeKeyBuffers;
var
  KeyIndex: TMemKeyIndex;
begin
  for KeyIndex := Low(TMemKeyIndex) to High(TMemKeyIndex) do begin
    DisposeMem(FKeyBuffers[KeyIndex], FKeyBufferSize);
  end;
end;

Hello,
unfortunately this change restores the memory leak of the kiLookup key buffer. Please try again.

I am going to share an interim hotfix with you (the one I used before installing your 9.2.103 release - it worked, although it does not correctly handle ranges - I do not use ranges at all).

Regards from Prague,
Jaroslav

uDAMemDataSet.pas // changed methods since 9.2.103 release's source

type
...
  TDAMemoryDataset = class(TDataset)
...
  protected
...
/// <CALLIDA>
    procedure DuplicateKeyBuffer(ASource, ADest: PMemKeyBuffer);
/// </CALLIDA>
...
/// <CALLIDA>
procedure TDAMemoryDataset.DuplicateKeyBuffer(ASource, ADest: PMemKeyBuffer);
var
  I: Integer;
  LSourceBuf, LDestBuf: Dataset_PByte;
  p, p2: PBlobRecord;
begin
  Move(ASource^, ADest^, SizeOf(TMemKeyBuffer));
  LSourceBuf:= Dataset_PByte(ASource) + SizeOf(TMemKeyBuffer);
  LDestBuf:= Dataset_PByte(ADest) + SizeOf(TMemKeyBuffer);
  if not GetHasReferencedFields then begin
    Move(LSourceBuf^, LDestBuf^, FNativeRecordSize);
  end
  else begin
    Move(LSourceBuf^, LDestBuf^, FNullMaskSize);

    for I := 0 to FieldCount - 1 do begin
      if (not GetNullMask(LSourceBuf, i)) then begin
        if not IsReferencedField(FDataTypeArray[i])  then begin
          Move(pointer(LSourceBuf + FOffsets[i])^, pointer(LDestBuf + FOffsets[i])^, FDataSizeArray[i])
        end
        else begin
          case FDataTypeArray[i] of
            ftString, ftFixedChar: PBytes(LDestBuf + FOffsets[i])^ := PBytes(LSourceBuf + FOffsets[i])^;
            {$IFDEF DA_FixedWideCharSupport}ftFixedWideChar,{$ENDIF DA_FixedWideCharSupport}
            ftWideString: PUnicodeString(LDestBuf + FOffsets[i])^ := PUnicodeString(LSourceBuf + FOffsets[i])^;
          else
            if FDataTypeArray[i] in ft_BlobTypes then begin
              p := PPointer(LSourceBuf + FOffsets[i])^;
              if p <> nil then begin
                p2 := CreateBlobRecord(PBlobRecord(p)^.size);
                Move(pointer(p)^, pointer(p2)^, p^.size + SizeOf(TBlobRecord));
                PPointer(LDestBuf + FOffsets[i])^ := p2;
              end
              else begin
                SetNullMask(LDestBuf, i, True);
              end;
            end;
          end;
        end;
      end;
    end;
  end;
end;
/// </CALLIDA>
...
procedure TDAMemoryDataset.SetKeyBuffer(KeyIndex: TMemKeyIndex; Clear: Boolean);
begin
  CheckBrowseMode;
  RefreshIndexConditional;
  FKeyBuffer := FKeyBuffers[KeyIndex];
/// <CALLIDA>
  // clear old saved key's data
  ClearBin2Buffer(Dataset_PByte(FKeyBuffers[kiSave]) + SizeOf(TMemKeyBuffer));
/// </CALLIDA>
  FillChar(FKeyBuffers[kiSave]^, FKeyBufferSize, 0);
  /// <CALLIDA>
  DuplicateKeyBuffer(FKeyBuffer, FKeyBuffers[kiSave]);
/// </CALLIDA>
  if Clear then InitKeyBuffer(FKeyBuffer);
  FKeyBuffer.FieldCount:= f_DefaultIndexRecord.IndexFieldNameList.Count;
  SetState(dsSetKey);
  SetModified(FKeyBuffer.Modified);
  DataEvent(deDataSetChange, 0);
end;
...
procedure TDAMemoryDataset.AllocKeyBuffers;
var
  KeyIndex: TMemKeyIndex;
begin
  try
    for KeyIndex := Low(TMemKeyIndex) to High(TMemKeyIndex) do begin
      FKeyBuffers[KeyIndex] := AllocMem(FKeyBufferSize);
      FillChar(FKeyBuffers[KeyIndex]^, FKeyBufferSize, 0);
    end;
    if Assigned(FCloneSource) and FCloneSource.Active then
      for KeyIndex := Low(TMemKeyIndex) to High(TMemKeyIndex) do
/// <CALLIDA>
        DuplicateKeyBuffer(FCloneSource.FKeyBuffers[KeyIndex], FKeyBuffers[KeyIndex]);
/// </CALLIDA>
  except
    FreeKeyBuffers;
    raise;
  end;
end;
...
procedure TDAMemoryDataset.FreeKeyBuffers;
var
  KeyIndex: TMemKeyIndex;
begin
  for KeyIndex := Low(TMemKeyIndex) to High(TMemKeyIndex) do begin
    ClearBin2Buffer(@FKeyBuffers[KeyIndex]^.Data);
    DisposeMem(FKeyBuffers[KeyIndex], FKeyBufferSize);
  end;
end;
...
procedure TDAMemoryDataset.PostKeyBuffer(Commit: Boolean);
begin
  DataEvent(deCheckBrowseMode, 0);
  if Commit then begin
    FKeyBuffer.Modified := Modified;
/// <CALLIDA>
    // release saved key's data
    ClearBin2Buffer(Dataset_PByte(FKeyBuffers[kiSave]) + SizeOf(TMemKeyBuffer));
/// </CALLIDA>
  end else begin
/// <CALLIDA>
    // clear current key's data
    ClearBin2Buffer(Dataset_PByte(FKeyBuffer) + SizeOf(TMemKeyBuffer));
    Move(FKeyBuffers[kiSave]^, FKeyBuffer^, FKeyBufferSize);
    FillChar(FKeyBuffers[kiSave]^, FKeyBufferSize, 0); // moved
/// </CALLIDA>
  end;
  SetState(dsBrowse);
  DataEvent(deDataSetChange, 0);
end;

thx, I’ll review your changes.
both your testcases, posted here, didn’t generate memory leaks with latest my fix

Thanks, logged as bugs://78150

bugs://78150 got closed with status fixed.