diff --git a/lib/csv/parser.rb b/lib/csv/parser.rb
index 25f10dfc..9b8a009b 100644
--- a/lib/csv/parser.rb
+++ b/lib/csv/parser.rb
@@ -978,63 +978,69 @@ def parse_no_quote(&block)
 
     def parse_quotable_loose(&block)
       @scanner.keep_start
-      @scanner.each_line(@row_separator) do |line|
-        if @skip_lines and skip_line?(line)
-          @scanner.keep_drop
-          @scanner.keep_start
-          next
-        end
-        original_line = line
-        line = line.delete_suffix(@row_separator)
-
-        if line.empty?
-          if @skip_blanks
+      kept = true
+      begin
+        @scanner.each_line(@row_separator) do |line|
+          if @skip_lines and skip_line?(line)
             @scanner.keep_drop
             @scanner.keep_start
             next
           end
-          row = []
-          quoted_fields = FieldsConverter::NO_QUOTED_FIELDS
-        elsif line.include?(@cr) or line.include?(@lf)
-          @scanner.keep_back
-          @parse_method = :parse_quotable_robust
-          return parse_quotable_robust(&block)
-        else
-          row = line.split(@split_column_separator, -1)
-          quoted_fields = []
-          n_columns = row.size
-          i = 0
-          while i < n_columns
-            column = row[i]
-            if column.empty?
-              quoted_fields << false
-              row[i] = nil
-            else
-              n_quotes = column.count(@quote_character)
-              if n_quotes.zero?
+          original_line = line
+          line = line.delete_suffix(@row_separator)
+
+          if line.empty?
+            if @skip_blanks
+              @scanner.keep_drop
+              @scanner.keep_start
+              next
+            end
+            row = []
+            quoted_fields = FieldsConverter::NO_QUOTED_FIELDS
+          elsif line.include?(@cr) or line.include?(@lf)
+            kept = false
+            @scanner.keep_back
+            @parse_method = :parse_quotable_robust
+            return parse_quotable_robust(&block)
+          else
+            row = line.split(@split_column_separator, -1)
+            quoted_fields = []
+            n_columns = row.size
+            i = 0
+            while i < n_columns
+              column = row[i]
+              if column.empty?
                 quoted_fields << false
-                # no quote
-              elsif n_quotes == 2 and
-                   column.start_with?(@quote_character) and
-                   column.end_with?(@quote_character)
-                quoted_fields << true
-                row[i] = column[1..-2]
+                row[i] = nil
               else
-                @scanner.keep_back
-                @parse_method = :parse_quotable_robust
-                return parse_quotable_robust(&block)
+                n_quotes = column.count(@quote_character)
+                if n_quotes.zero?
+                  quoted_fields << false
+                  # no quote
+                elsif n_quotes == 2 and
+                     column.start_with?(@quote_character) and
+                     column.end_with?(@quote_character)
+                  quoted_fields << true
+                  row[i] = column[1..-2]
+                else
+                  kept = false
+                  @scanner.keep_back
+                  @parse_method = :parse_quotable_robust
+                  return parse_quotable_robust(&block)
+                end
+                validate_field_size(row[i])
               end
-              validate_field_size(row[i])
+              i += 1
             end
-            i += 1
           end
+          @scanner.keep_drop
+          @scanner.keep_start
+          @last_line = original_line
+          emit_row(row, quoted_fields, &block)
         end
-        @scanner.keep_drop
-        @scanner.keep_start
-        @last_line = original_line
-        emit_row(row, quoted_fields, &block)
+      ensure
+        @scanner.keep_drop if kept
       end
-      @scanner.keep_drop
     end
 
     def parse_quotable_robust(&block)
diff --git a/test/csv/parse/test_read.rb b/test/csv/parse/test_read.rb
index ba6fe985..f0508406 100644
--- a/test/csv/parse/test_read.rb
+++ b/test/csv/parse/test_read.rb
@@ -24,4 +24,34 @@ def test_shift
                    csv.shift,
                  ])
   end
+
+  def test_shift_keeps
+    csv = CSV.new("a,b\n" * 3)
+    assert_equal([
+                   ["a", "b"],
+                   ["a", "b"],
+                   0,
+                 ],
+                 [
+                   csv.shift,
+                   csv.shift,
+                   scanner_keeps(csv).size,
+                 ])
+  end
+
+  def test_shift_keeps_after_malformed
+    csv = CSV.new("a,b\nabcde,b\n", max_field_size: 4)
+    assert_equal(["a", "b"], csv.shift)
+    assert_raise(CSV::MalformedCSVError) do
+      csv.shift
+    end
+    assert_equal(0, scanner_keeps(csv).size)
+  end
+
+  private
+  def scanner_keeps(csv)
+    parser = csv.instance_variable_get(:@parser)
+    scanner = parser.instance_variable_get(:@scanner)
+    scanner.instance_variable_get(:@keeps)
+  end
 end