How to drag a MapPolygon on a Map

I am trying to drag a MapPolygon on a SwiftUI Map. I have created this example code,
to show it is not dragging the MapPolygon smoothly and accurately.

What is a more appropriate way to drag a whole MapPolygon on a Map smoothly and accurately?

The problem as I see it, is the translation of all points of the MapPolygon (doDrag),
this is not a good way to do translation, but I cannot find a better one.

(ios-17 and MacCatalyst)

struct ContentView: View {
    @State private var isDraging = false
    
    var body: some View {
        VStack {
            ToolbarView(isDraging: $isDraging)
            MapViewer(isDraging: $isDraging)
        }
    }
}

struct MapViewer: View {
    @Binding var isDraging: Bool
    @State private var modes = MapInteractionModes.all
    
    @State private var cameraPosition: MapCameraPosition = .camera(
        MapCamera(centerCoordinate: CLLocationCoordinate2D(latitude: 35.68, longitude: 139.75), distance: 4000.0, heading: 0, pitch: 0)
    )
    
    @State private var polyPoints = [CLLocationCoordinate2D(latitude: 35.682890728577135, longitude: 139.74688521853778),CLLocationCoordinate2D(latitude: 35.682457291364656, longitude: 139.7513297533676),CLLocationCoordinate2D(latitude: 35.679605242840324, longitude: 139.74859855588522)]
    
    var body: some View {
        MapReader { reader in
            Map(position: $cameraPosition, interactionModes: modes) {
                MapPolygon(coordinates: polyPoints)
                    .stroke(.white, lineWidth: 2)
                    .foregroundStyle(.purple.opacity(0.3))
            }
            .gesture(DragGesture()
                .onChanged { drag in
                    if isDraging {
                        doDrag(drag)
                    }
                }
            )
        }
        .mapStyle(.imagery)
        .mapControlVisibility(.hidden)
        .edgesIgnoringSafeArea(.all)
        .onChange(of: isDraging) {
            if isDraging { modes.subtract(.all) }
            else { modes.update(with: .all) }
        }
    }
    
    // this is not working well
    func doDrag(_ drag: DragGesture.Value) {
        for i in polyPoints.indices {
            let p = MKMapPoint(polyPoints[i])
            let newP = MKMapPoint(x: p.x + drag.translation.width, y: p.y + drag.translation.height)
            polyPoints[i] = newP.coordinate
        }
    }
 
}

struct ToolbarView: View {
    @Binding var isDraging: Bool
    
    var body: some View {
        Button(action: { isDraging.toggle() }) {
            VStack {
                Image(systemName: "move.3d")
                    .resizable()
                    .frame(width: 25, height: 25)
                    .foregroundColor(isDraging ? .red : .blue)
                Text(isDraging ? "Drag on" : "Drag off")
                    .font(.caption)
                    .foregroundColor(isDraging ? .red : .blue)
            }.frame(width: 80, height: 60)
        }
        .buttonStyle(.bordered)
        .background(Color.white)
        .clipShape(RoundedRectangle(cornerRadius: 10))
    }
}

Tried to translate all points of the MapPolygon, but that does not give a acceptable results.